diff --git a/.coveragerc b/.coveragerc index 91c9b36bc3..70339edd4d 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,7 @@ [run] branch = True -source = flaml +source = + flaml omit = - *test* + */test/* + */flaml/autogen/* diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml index 0ea5058bdd..f390099f55 100644 --- a/.github/workflows/CD.yml +++ b/.github/workflows/CD.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest"] - python-version: ["3.10"] + python-version: ["3.12"] runs-on: ${{ matrix.os }} environment: package steps: diff --git a/.github/workflows/deploy-website.yml b/.github/workflows/deploy-website.yml index 26d3d129b9..2b02240400 100644 --- a/.github/workflows/deploy-website.yml +++ b/.github/workflows/deploy-website.yml @@ -37,11 +37,11 @@ jobs: - name: setup python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.12" - name: pydoc-markdown install run: | python -m pip install --upgrade pip - pip install pydoc-markdown==4.7.0 + pip install pydoc-markdown==4.7.0 setuptools - name: pydoc-markdown run run: | pydoc-markdown @@ -73,11 +73,11 @@ jobs: - name: setup python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.12" - name: pydoc-markdown install run: | python -m pip install --upgrade pip - pip install pydoc-markdown==4.7.0 + pip install pydoc-markdown==4.7.0 setuptools - name: pydoc-markdown run run: | pydoc-markdown diff --git a/.github/workflows/openai.yml b/.github/workflows/openai.yml index c7dc0f5490..e85569ed4e 100644 --- a/.github/workflows/openai.yml +++ b/.github/workflows/openai.yml @@ -4,14 +4,15 @@ name: OpenAI on: - pull_request: - branches: ['main'] - paths: - - 'flaml/autogen/**' - - 'test/autogen/**' - - 'notebook/autogen_openai_completion.ipynb' - - 'notebook/autogen_chatgpt_gpt4.ipynb' - - '.github/workflows/openai.yml' + workflow_dispatch: +# pull_request: +# branches: ['main'] +# paths: +# - 'flaml/autogen/**' +# - 'test/autogen/**' +# - 'notebook/autogen_openai_completion.ipynb' +# - 'notebook/autogen_chatgpt_gpt4.ipynb' +# - '.github/workflows/openai.yml' permissions: {} diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 9e88a4e8da..0098f5529b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -40,10 +40,12 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.10", "3.11"] + python-version: ["3.10", "3.11", "3.12"] exclude: - os: macos-latest - python-version: "3.10" + python-version: "3.10" # macOS runners will hang on python 3.10 for unknown reasons + - os: macos-latest + python-version: "3.12" # macOS runners will hang on python 3.12 for unknown reasons steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -67,11 +69,6 @@ jobs: pip install -e . python -c "import flaml" pip install -e .[test] - - name: On Ubuntu python 3.10, install pyspark 3.4.1 - if: matrix.python-version == '3.10' && matrix.os == 'ubuntu-latest' - run: | - pip install pyspark==3.4.1 - pip list | grep "pyspark" - name: On Ubuntu python 3.11, install pyspark 3.5.1 if: matrix.python-version == '3.11' && matrix.os == 'ubuntu-latest' run: | @@ -106,17 +103,17 @@ jobs: run: | pip cache purge - name: Test with pytest - if: matrix.python-version != '3.10' + if: matrix.python-version != '3.11' run: | pytest test/ --ignore=test/autogen --reruns 2 --reruns-delay 10 - name: Coverage - if: matrix.python-version == '3.10' + if: matrix.python-version == '3.11' run: | pip install coverage coverage run -a -m pytest test --ignore=test/autogen --reruns 2 --reruns-delay 10 coverage xml - name: Upload coverage to Codecov - if: matrix.python-version == '3.10' + if: matrix.python-version == '3.11' uses: codecov/codecov-action@v3 with: file: ./coverage.xml @@ -130,10 +127,7 @@ jobs: BRANCH=unit-tests-installed-dependencies git fetch origin - git checkout -B "$BRANCH" - if git show-ref --verify --quiet "refs/remotes/origin/$BRANCH"; then - git rebase "origin/$BRANCH" - fi + git checkout -B "$BRANCH" "origin/$BRANCH" pip freeze > installed_all_dependencies_${{ matrix.python-version }}_${{ matrix.os }}.txt python test/check_dependency.py > installed_first_tier_dependencies_${{ matrix.python-version }}_${{ matrix.os }}.txt @@ -141,4 +135,4 @@ jobs: mv coverage.xml ./coverage_${{ matrix.python-version }}_${{ matrix.os }}.xml || true git add -f ./coverage_${{ matrix.python-version }}_${{ matrix.os }}.xml || true git commit -m "Update installed dependencies for Python ${{ matrix.python-version }} on ${{ matrix.os }}" || exit 0 - git push origin "$BRANCH" + git push origin "$BRANCH" --force diff --git a/.gitignore b/.gitignore index 18c858dad2..0e7cf96e90 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,7 @@ coverage.xml .hypothesis/ .pytest_cache/ cover/ +junit # Translations *.mo diff --git a/flaml/autogen/__init__.py b/flaml/autogen/__init__.py index 7a557e081e..02e6ff7d24 100644 --- a/flaml/autogen/__init__.py +++ b/flaml/autogen/__init__.py @@ -1,3 +1,12 @@ +import warnings + from .agentchat import * from .code_utils import DEFAULT_MODEL, FAST_MODEL from .oai import * + +warnings.warn( + "The `flaml.autogen` module is deprecated and will be removed in a future release. " + "Please refer to `https://github.com/microsoft/autogen` for latest usage.", + DeprecationWarning, + stacklevel=2, +) diff --git a/flaml/automl/automl.py b/flaml/automl/automl.py index c4c6e2dbb5..5cf2d71727 100644 --- a/flaml/automl/automl.py +++ b/flaml/automl/automl.py @@ -4,6 +4,7 @@ # * project root for license information. from __future__ import annotations +import inspect import json import logging import os @@ -177,10 +178,11 @@ def custom_metric( ['auto', 'cv', 'holdout']. split_ratio: A float of the valiation data percentage for holdout. n_splits: An integer of the number of folds for cross - validation. - log_type: A string of the log type, one of - ['better', 'all']. - 'better' only logs configs with better loss than previos iters - 'all' logs all the tried configs. + log_type: Specifies which logs to save. One of ['better', 'all']. Default is 'better'. + - 'better': Logs configs and models (if `model_history` is True) only when the loss improves, + to `log_file_name` and MLflow, respectively. + - 'all': Logs all configs and models (if `model_history` is True), regardless of performance. + Note: Configs are always logged to MLflow if MLflow logging is enabled. model_history: A boolean of whether to keep the best model per estimator. Make sure memory is large enough if setting to True. Default False. log_training_metric: A boolean of whether to log the training @@ -2174,7 +2176,7 @@ def _search_parallel(self): use_spark=True, force_cancel=self._force_cancel, mlflow_exp_name=self._mlflow_exp_name, - automl_info=(mlflow_log_latency,), # pass automl info to tune.run + automl_info=(mlflow_log_latency, self._log_type), # pass automl info to tune.run extra_tag=self.autolog_extra_tag, # raise_on_failed_trial=False, # keep_checkpoints_num=1, @@ -2237,7 +2239,9 @@ def _search_parallel(self): if better or self._log_type == "all": self._log_trial(search_state, estimator) if self.mlflow_integration: - self.mlflow_integration.record_state(self, search_state, estimator) + self.mlflow_integration.record_state( + self, search_state, estimator, better or self._log_type == "all" + ) def _log_trial(self, search_state, estimator): if self._training_log: @@ -2479,7 +2483,9 @@ def _search_sequential(self): if better or self._log_type == "all": self._log_trial(search_state, estimator) if self.mlflow_integration: - self.mlflow_integration.record_state(self, search_state, estimator) + self.mlflow_integration.record_state( + self, search_state, estimator, better or self._log_type == "all" + ) logger.info( " at {:.1f}s,\testimator {}'s best error={:.4f},\tbest estimator {}'s best error={:.4f}".format( diff --git a/flaml/automl/data.py b/flaml/automl/data.py index 096ba46d90..53645a3584 100644 --- a/flaml/automl/data.py +++ b/flaml/automl/data.py @@ -5,6 +5,7 @@ import json import os import random +import re import uuid from datetime import datetime, timedelta from decimal import ROUND_HALF_UP, Decimal @@ -708,6 +709,14 @@ def auto_convert_dtypes_pandas( """ if na_values is None: na_values = {"NA", "na", "NULL", "null", ""} + # Remove the empty string separately (handled by the regex `^\s*$`) + vals = [re.escape(v) for v in na_values if v != ""] + # Build inner alternation group + inner = "|".join(vals) if vals else "" + if inner: + pattern = re.compile(rf"^\s*(?:{inner})?\s*$") + else: + pattern = re.compile(r"^\s*$") df_converted = df.convert_dtypes() schema = {} @@ -721,7 +730,11 @@ def auto_convert_dtypes_pandas( for col in df.columns: series = df[col] # Replace NA-like values if string - series_cleaned = series.map(lambda x: np.nan if isinstance(x, str) and x.strip() in na_values else x) + if series.dtype == object: + mask = series.astype(str).str.match(pattern) + series_cleaned = series.where(~mask, np.nan) + else: + series_cleaned = series # Skip conversion if already non-object data type, except bool which can potentially be categorical if ( diff --git a/flaml/automl/model.py b/flaml/automl/model.py index 53a92ece24..99fd9c0c61 100644 --- a/flaml/automl/model.py +++ b/flaml/automl/model.py @@ -2347,8 +2347,11 @@ def config2params(self, config: dict) -> dict: params = super().config2params(config) params["tol"] = params.get("tol", 0.0001) params["loss"] = params.get("loss", None) - if params["loss"] is None and self._task.is_classification(): - params["loss"] = "log_loss" if SKLEARN_VERSION >= "1.1" else "log" + if params["loss"] is None: + if self._task.is_classification(): + params["loss"] = "log_loss" if SKLEARN_VERSION >= "1.1" else "log" + else: + params["loss"] = "squared_error" if not self._task.is_classification() and "n_jobs" in params: params.pop("n_jobs") diff --git a/flaml/automl/spark/__init__.py b/flaml/automl/spark/__init__.py index c8c8111f16..e95d089a2f 100644 --- a/flaml/automl/spark/__init__.py +++ b/flaml/automl/spark/__init__.py @@ -1,3 +1,5 @@ +import atexit +import logging import os os.environ["PYARROW_IGNORE_TIMEZONE"] = "1" @@ -10,13 +12,14 @@ from pyspark.pandas import Series as psSeries from pyspark.pandas import set_option from pyspark.sql import DataFrame as sparkDataFrame + from pyspark.sql import SparkSession from pyspark.util import VersionUtils except ImportError: class psDataFrame: pass - F = T = ps = sparkDataFrame = psSeries = psDataFrame + F = T = ps = sparkDataFrame = SparkSession = psSeries = psDataFrame _spark_major_minor_version = set_option = None ERROR = ImportError( """Please run pip install flaml[spark] @@ -32,3 +35,60 @@ class psDataFrame: from pandas import DataFrame, Series except ImportError: DataFrame = Series = pd = None + + +logger = logging.getLogger(__name__) + + +def disable_spark_ansi_mode(): + """Disable Spark ANSI mode if it is enabled.""" + spark = SparkSession.getActiveSession() if hasattr(SparkSession, "getActiveSession") else None + adjusted = False + try: + ps_conf = ps.get_option("compute.fail_on_ansi_mode") + except Exception: + ps_conf = None + ansi_conf = [None, ps_conf] # ansi_conf and ps_conf original values + # Spark may store the config as string 'true'/'false' (or boolean in some contexts) + if spark is not None: + ansi_conf[0] = spark.conf.get("spark.sql.ansi.enabled") + ansi_enabled = ( + (isinstance(ansi_conf[0], str) and ansi_conf[0].lower() == "true") + or (isinstance(ansi_conf[0], bool) and ansi_conf[0] is True) + or ansi_conf[0] is None + ) + try: + if ansi_enabled: + logger.debug("Adjusting spark.sql.ansi.enabled to false") + spark.conf.set("spark.sql.ansi.enabled", "false") + adjusted = True + except Exception: + # If reading/setting options fail for some reason, keep going and let + # pandas-on-Spark raise a meaningful error later. + logger.exception("Failed to set spark.sql.ansi.enabled") + + if ansi_conf[1]: + logger.debug("Adjusting pandas-on-Spark compute.fail_on_ansi_mode to False") + ps.set_option("compute.fail_on_ansi_mode", False) + adjusted = True + + return spark, ansi_conf, adjusted + + +def restore_spark_ansi_mode(spark, ansi_conf, adjusted): + """Restore Spark ANSI mode to its original setting.""" + # Restore the original spark.sql.ansi.enabled to avoid persistent side-effects. + if adjusted and spark and ansi_conf[0] is not None: + try: + logger.debug(f"Restoring spark.sql.ansi.enabled to {ansi_conf[0]}") + spark.conf.set("spark.sql.ansi.enabled", ansi_conf[0]) + except Exception: + logger.exception("Failed to restore spark.sql.ansi.enabled") + + if adjusted and ansi_conf[1]: + logger.debug(f"Restoring pandas-on-Spark compute.fail_on_ansi_mode to {ansi_conf[1]}") + ps.set_option("compute.fail_on_ansi_mode", ansi_conf[1]) + + +spark, ansi_conf, adjusted = disable_spark_ansi_mode() +atexit.register(restore_spark_ansi_mode, spark, ansi_conf, adjusted) diff --git a/flaml/automl/spark/utils.py b/flaml/automl/spark/utils.py index 5f3a2a606e..9a902e014a 100644 --- a/flaml/automl/spark/utils.py +++ b/flaml/automl/spark/utils.py @@ -59,17 +59,29 @@ def to_pandas_on_spark( ``` """ set_option("compute.default_index_type", default_index_type) - if isinstance(df, (DataFrame, Series)): - return ps.from_pandas(df) - elif isinstance(df, sparkDataFrame): - if _spark_major_minor_version[0] == 3 and _spark_major_minor_version[1] < 3: - return df.to_pandas_on_spark(index_col=index_col) + try: + orig_ps_conf = ps.get_option("compute.fail_on_ansi_mode") + except Exception: + orig_ps_conf = None + if orig_ps_conf: + ps.set_option("compute.fail_on_ansi_mode", False) + + try: + if isinstance(df, (DataFrame, Series)): + return ps.from_pandas(df) + elif isinstance(df, sparkDataFrame): + if _spark_major_minor_version[0] == 3 and _spark_major_minor_version[1] < 3: + return df.to_pandas_on_spark(index_col=index_col) + else: + return df.pandas_api(index_col=index_col) + elif isinstance(df, (psDataFrame, psSeries)): + return df else: - return df.pandas_api(index_col=index_col) - elif isinstance(df, (psDataFrame, psSeries)): - return df - else: - raise TypeError(f"{type(df)} is not one of pandas.DataFrame, pandas.Series and pyspark.sql.DataFrame") + raise TypeError(f"{type(df)} is not one of pandas.DataFrame, pandas.Series and pyspark.sql.DataFrame") + finally: + # Restore original config + if orig_ps_conf: + ps.set_option("compute.fail_on_ansi_mode", orig_ps_conf) def train_test_split_pyspark( diff --git a/flaml/default/greedy.py b/flaml/default/greedy.py index d03f971aac..2cb5d67e34 100644 --- a/flaml/default/greedy.py +++ b/flaml/default/greedy.py @@ -32,6 +32,7 @@ def construct_portfolio(regret_matrix, meta_features, regret_bound): if meta_features is not None: scaler = RobustScaler() meta_features = meta_features.loc[tasks] + meta_features = meta_features.astype(float) meta_features.loc[:, :] = scaler.fit_transform(meta_features) nearest_task = {} for t in tasks: diff --git a/flaml/default/portfolio.py b/flaml/default/portfolio.py index 2de856523e..c2c971d8b0 100644 --- a/flaml/default/portfolio.py +++ b/flaml/default/portfolio.py @@ -26,6 +26,7 @@ def config_predictor_tuple(tasks, configs, meta_features, regret_matrix): # pre-processing scaler = RobustScaler() meta_features_norm = meta_features.loc[tasks] # this makes a copy + meta_features_norm = meta_features_norm.astype(float) meta_features_norm.loc[:, :] = scaler.fit_transform(meta_features_norm) proc = { diff --git a/flaml/fabric/mlflow.py b/flaml/fabric/mlflow.py index ffcbe71d96..05ded1fa14 100644 --- a/flaml/fabric/mlflow.py +++ b/flaml/fabric/mlflow.py @@ -567,7 +567,7 @@ def _pickle_and_log_artifact(self, obj, artifact_name, pickle_fname="temp_.pkl", try: with open(pickle_fpath, "wb") as f: pickle.dump(obj, f) - mlflow.log_artifact(pickle_fpath, artifact_name, run_id) + self.mlflow_client.log_artifact(run_id, pickle_fpath, artifact_name) return True except Exception as e: logger.debug(f"Failed to pickle and log {artifact_name}, error: {e}") @@ -652,7 +652,7 @@ def pickle_and_log_automl_artifacts(self, automl, model, estimator, signature=No return f"Successfully pickle_and_log_automl_artifacts {estimator} to run_id {run_id}" @time_it - def record_state(self, automl, search_state, estimator): + def record_state(self, automl, search_state, estimator, is_log_model=True): _st = time.time() automl_metric_name = ( automl._state.metric if isinstance(automl._state.metric, str) else automl._state.error_metric @@ -727,7 +727,7 @@ def record_state(self, automl, search_state, estimator): self.futures[future] = f"iter_{automl._track_iter}_log_info_to_run" future = executor.submit(lambda: self._log_automl_configurations(child_run.info.run_id)) self.futures[future] = f"iter_{automl._track_iter}_log_automl_configurations" - if automl._state.model_history: + if automl._state.model_history and is_log_model: if estimator.endswith("_spark"): future = executor.submit( lambda: self.log_model( @@ -797,8 +797,10 @@ def log_automl(self, automl): conf = automl._config_history[automl._best_iteration][1].copy() if "ml" in conf.keys(): conf = conf["ml"] - - mlflow.log_params({**conf, "best_learner": automl._best_estimator}, run_id=self.parent_run_id) + params_arr = [ + Param(key, str(value)) for key, value in {**conf, "best_learner": automl._best_estimator}.items() + ] + self.mlflow_client.log_batch(run_id=self.parent_run_id, metrics=[], params=params_arr, tags=[]) if not self.has_summary: logger.info(f"logging best model {automl.best_estimator}") future = executor.submit(lambda: self.copy_mlflow_run(best_mlflow_run_id, self.parent_run_id)) @@ -894,6 +896,7 @@ def adopt_children(self, result=None): ), ) self.child_counter = 0 + num_infos = len(self.infos) # From latest to earliest, remove duplicate cross-validation runs _exist_child_run_params = [] # for deduplication of cross-validation child runs @@ -958,22 +961,37 @@ def adopt_children(self, result=None): ) self.mlflow_client.set_tag(child_run_id, "flaml.child_counter", self.child_counter) - # merge autolog child run and corresponding manual run - flaml_info = self.infos[self.child_counter] - child_run = self.mlflow_client.get_run(child_run_id) - self._log_info_to_run(flaml_info, child_run_id, log_params=False) - - if self.experiment_type == "automl": - if "learner" not in child_run.data.params: - self.mlflow_client.log_param(child_run_id, "learner", flaml_info["params"]["learner"]) - if "sample_size" not in child_run.data.params: - self.mlflow_client.log_param( - child_run_id, "sample_size", flaml_info["params"]["sample_size"] - ) + # Merge autolog child run and corresponding FLAML trial info (if available). + # In nested scenarios (e.g., Tune -> AutoML -> MLflow autolog), MLflow can create + # more child runs than the number of FLAML trials recorded in self.infos. + # TODO: need more tests in nested scenarios. + flaml_info = None + child_run = None + if self.child_counter < num_infos: + flaml_info = self.infos[self.child_counter] + child_run = self.mlflow_client.get_run(child_run_id) + self._log_info_to_run(flaml_info, child_run_id, log_params=False) + + if self.experiment_type == "automl": + if "learner" not in child_run.data.params: + self.mlflow_client.log_param(child_run_id, "learner", flaml_info["params"]["learner"]) + if "sample_size" not in child_run.data.params: + self.mlflow_client.log_param( + child_run_id, "sample_size", flaml_info["params"]["sample_size"] + ) + else: + logger.debug( + "No corresponding FLAML info for MLflow child run %s (child_counter=%s, infos=%s); skipping merge.", + child_run_id, + self.child_counter, + num_infos, + ) - if self.child_counter == best_iteration: + if flaml_info is not None and self.child_counter == best_iteration: self.mlflow_client.set_tag(child_run_id, "flaml.best_run", True) if result is not None: + if child_run is None: + child_run = self.mlflow_client.get_run(child_run_id) result.best_run_id = child_run_id result.best_run_name = child_run.info.run_name self.best_run_id = child_run_id @@ -997,7 +1015,7 @@ def __del__(self): self.resume_mlflow() -def register_automl_pipeline(automl, model_name=None, signature=None): +def register_automl_pipeline(automl, model_name=None, signature=None, artifact_path="model"): pipeline = automl.automl_pipeline if pipeline is None: logger.warning("pipeline not found, cannot register it") @@ -1007,7 +1025,7 @@ def register_automl_pipeline(automl, model_name=None, signature=None): if automl.best_run_id is None: mlflow.sklearn.log_model( pipeline, - "automl_pipeline", + artifact_path, registered_model_name=model_name, signature=automl.pipeline_signature if signature is None else signature, ) @@ -1017,5 +1035,5 @@ def register_automl_pipeline(automl, model_name=None, signature=None): return mvs[0] else: best_run = mlflow.get_run(automl.best_run_id) - model_uri = f"runs:/{best_run.info.run_id}/automl_pipeline" + model_uri = f"runs:/{best_run.info.run_id}/{artifact_path}" return mlflow.register_model(model_uri, model_name) diff --git a/flaml/tune/space.py b/flaml/tune/space.py index 0f4f6f8eac..5be42fd4fc 100644 --- a/flaml/tune/space.py +++ b/flaml/tune/space.py @@ -261,7 +261,7 @@ def add_cost_to_space(space: Dict, low_cost_point: Dict, choice_cost: Dict): low_cost[i] = point if len(low_cost) > len(domain.categories): if domain.ordered: - low_cost[-1] = int(np.where(ind == low_cost[-1])[0]) + low_cost[-1] = int(np.where(ind == low_cost[-1])[0].item()) domain.low_cost_point = low_cost[-1] return if low_cost: diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py index 08a45c72c1..2be9dc858a 100644 --- a/flaml/tune/tune.py +++ b/flaml/tune/tune.py @@ -776,7 +776,7 @@ def easy_objective(config): and (num_samples < 0 or num_trials < num_samples) and num_failures < upperbound_num_failures ): - if automl_info and automl_info[0] > 0 and time_budget_s < np.inf: + if automl_info and automl_info[1] == "all" and automl_info[0] > 0 and time_budget_s < np.inf: time_budget_s -= automl_info[0] * n_concurrent_trials logger.debug(f"Remaining time budget with mlflow log latency: {time_budget_s} seconds.") while len(_runner.running_trials) < n_concurrent_trials: @@ -802,9 +802,17 @@ def easy_objective(config): ) results = None with PySparkOvertimeMonitor(time_start, time_budget_s, force_cancel, parallel=parallel): - results = parallel( - delayed(evaluation_function)(trial_to_run.config) for trial_to_run in trials_to_run - ) + try: + results = parallel( + delayed(evaluation_function)(trial_to_run.config) for trial_to_run in trials_to_run + ) + except RuntimeError as e: + logger.warning(f"RuntimeError: {e}") + results = None + logger.info( + "Encountered RuntimeError. Waiting 10 seconds for Spark cluster to recover before retrying." + ) + time.sleep(10) # results = [evaluation_function(trial_to_run.config) for trial_to_run in trials_to_run] while results: result = results.pop(0) diff --git a/pyproject.toml b/pyproject.toml index 964fa46dd6..33cf99d1df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,6 @@ license_file = "LICENSE" description-file = "README.md" - [tool.pytest.ini_options] addopts = '-m "not conda"' markers = [ diff --git a/test/automl/test_custom_hp.py b/test/automl/test_custom_hp.py index bef6db823e..b06ae9f2c7 100644 --- a/test/automl/test_custom_hp.py +++ b/test/automl/test_custom_hp.py @@ -4,8 +4,17 @@ from flaml import AutoML, tune +try: + import transformers -@pytest.mark.skipif(sys.platform == "darwin", reason="do not run on mac os") + _transformers_installed = True +except ImportError: + _transformers_installed = False + + +@pytest.mark.skipif( + sys.platform == "darwin" or not _transformers_installed, reason="do not run on mac os or transformers not installed" +) def test_custom_hp_nlp(): from test.nlp.utils import get_automl_settings, get_toy_data_seqclassification diff --git a/test/automl/test_extra_models.py b/test/automl/test_extra_models.py index e19f2cb201..651737a410 100644 --- a/test/automl/test_extra_models.py +++ b/test/automl/test_extra_models.py @@ -1,3 +1,4 @@ +import atexit import os import sys import unittest @@ -15,8 +16,16 @@ from flaml import AutoML from flaml.automl.ml import sklearn_metric_loss_score +from flaml.automl.spark import disable_spark_ansi_mode, restore_spark_ansi_mode from flaml.tune.spark.utils import check_spark +try: + import pytorch_lightning + + _pl_installed = True +except ImportError: + _pl_installed = False + pytestmark = pytest.mark.spark leaderboard = defaultdict(dict) @@ -39,7 +48,7 @@ .config( "spark.jars.packages", ( - "com.microsoft.azure:synapseml_2.12:1.0.2," + "com.microsoft.azure:synapseml_2.12:1.1.0," "org.apache.hadoop:hadoop-azure:3.3.5," "com.microsoft.azure:azure-storage:8.6.6," f"org.mlflow:mlflow-spark_2.12:{mlflow.__version__}" @@ -63,6 +72,9 @@ except ImportError: skip_spark = True +spark, ansi_conf, adjusted = disable_spark_ansi_mode() +atexit.register(restore_spark_ansi_mode, spark, ansi_conf, adjusted) + def _test_regular_models(estimator_list, task): if isinstance(estimator_list, str): @@ -271,7 +283,11 @@ def test_aft(self): @unittest.skipIf(skip_spark, reason="Spark is not installed. Skip all spark tests.") def test_default_spark(self): - _test_spark_models(None, "classification") + # TODO: remove the estimator assignment once SynapseML supports spark 4+. + from flaml.automl.spark.utils import _spark_major_minor_version + + estimator_list = ["rf_spark"] if _spark_major_minor_version[0] >= 4 else None + _test_spark_models(estimator_list, "classification") def test_svc(self): _test_regular_models("svc", "classification") @@ -302,7 +318,7 @@ def test_seasonal_avg(self): def test_avg(self): _test_forecast("avg") - @unittest.skipIf(skip_spark, reason="Skip on Mac or Windows") + @unittest.skipIf(skip_spark or not _pl_installed, reason="Skip on Mac or Windows or no pytorch_lightning.") def test_tcn(self): _test_forecast("tcn") diff --git a/test/automl/test_forecast.py b/test/automl/test_forecast.py index 21e8bacfe3..49a9823172 100644 --- a/test/automl/test_forecast.py +++ b/test/automl/test_forecast.py @@ -10,7 +10,7 @@ from flaml.automl.task.time_series_task import TimeSeriesTask -def test_forecast_automl(budget=10, estimators_when_no_prophet=["arima", "sarimax", "holt-winters"]): +def test_forecast_automl(budget=20, estimators_when_no_prophet=["arima", "sarimax", "holt-winters"]): # using dataframe import statsmodels.api as sm @@ -510,8 +510,12 @@ def get_stalliion_data(): "3.11" in sys.version, reason="do not run on py 3.11", ) -def test_forecast_panel(budget=5): - data, special_days = get_stalliion_data() +def test_forecast_panel(budget=30): + try: + data, special_days = get_stalliion_data() + except ImportError: + print("pytorch_forecasting not installed") + return time_horizon = 6 # predict six months training_cutoff = data["time_idx"].max() - time_horizon data["time_idx"] = data["time_idx"].astype("int") diff --git a/test/automl/test_notebook_example.py b/test/automl/test_notebook_example.py index b4558f1092..06237373bd 100644 --- a/test/automl/test_notebook_example.py +++ b/test/automl/test_notebook_example.py @@ -79,6 +79,9 @@ def test_automl(budget=5, dataset_format="dataframe", hpo_method=None): automl.fit(X_train=X_train, y_train=y_train, **settings) """ retrieve best config and best learner """ print("Best ML leaner:", automl.best_estimator) + if not automl.best_estimator: + print("Training budget is not sufficient") + return print("Best hyperparmeter config:", automl.best_config) print(f"Best accuracy on validation data: {1 - automl.best_loss:.4g}") print(f"Training duration of best run: {automl.best_config_train_time:.4g} s") diff --git a/test/nlp/test_autohf_classificationhead.py b/test/nlp/test_autohf_classificationhead.py index 325e4aeb02..31b13d2270 100644 --- a/test/nlp/test_autohf_classificationhead.py +++ b/test/nlp/test_autohf_classificationhead.py @@ -3,6 +3,12 @@ import sys import pytest + +try: + import transformers +except ImportError: + pytest.skip("transformers not installed", allow_module_level=True) + from utils import ( get_automl_settings, get_toy_data_binclassification, diff --git a/test/nlp/test_autohf_cv.py b/test/nlp/test_autohf_cv.py index 4ec6113070..b95de2a9c1 100644 --- a/test/nlp/test_autohf_cv.py +++ b/test/nlp/test_autohf_cv.py @@ -5,10 +5,20 @@ import pytest from utils import get_automl_settings, get_toy_data_seqclassification +try: + import transformers + + _transformers_installed = True +except ImportError: + _transformers_installed = False + pytestmark = pytest.mark.spark # set to spark as parallel testing raised MlflowException of changing parameter -@pytest.mark.skipif(sys.platform in ["darwin", "win32"], reason="do not run on mac os or windows") +@pytest.mark.skipif( + sys.platform in ["darwin", "win32"] or not _transformers_installed, + reason="do not run on mac os or windows or transformers not installed", +) def test_cv(): import requests diff --git a/test/nlp/test_autohf_multichoice_classification.py b/test/nlp/test_autohf_multichoice_classification.py index a9983b39e1..0c90c25ea6 100644 --- a/test/nlp/test_autohf_multichoice_classification.py +++ b/test/nlp/test_autohf_multichoice_classification.py @@ -5,8 +5,18 @@ import pytest from utils import get_automl_settings, get_toy_data_multiplechoiceclassification +try: + import transformers -@pytest.mark.skipif(sys.platform in ["darwin", "win32"], reason="do not run on mac os or windows") + _transformers_installed = True +except ImportError: + _transformers_installed = False + + +@pytest.mark.skipif( + sys.platform in ["darwin", "win32"] or not _transformers_installed, + reason="do not run on mac os or windows or transformers not installed", +) def test_mcc(): import requests diff --git a/test/nlp/test_default.py b/test/nlp/test_default.py index 5f7622a1c9..ddd7cb8273 100644 --- a/test/nlp/test_default.py +++ b/test/nlp/test_default.py @@ -7,8 +7,20 @@ from flaml.default import portfolio -if sys.platform.startswith("darwin") and sys.version_info[0] == 3 and sys.version_info[1] == 11: - pytest.skip("skipping Python 3.11 on MacOS", allow_module_level=True) +try: + import transformers + + _transformers_installed = True +except ImportError: + _transformers_installed = False + +if ( + sys.platform.startswith("darwin") + and sys.version_info >= (3, 11) + or not _transformers_installed + or sys.platform == "win32" +): + pytest.skip("skipping Python 3.11 on MacOS or without transformers or on Windows", allow_module_level=True) pytestmark = ( pytest.mark.spark @@ -28,7 +40,6 @@ def test_build_portfolio(path="./test/nlp/default", strategy="greedy"): portfolio.main() -@pytest.mark.skipif(sys.platform == "win32", reason="do not run on windows") def test_starting_point_not_in_search_space(): """Regression test for invalid starting points and custom_hp. @@ -126,7 +137,6 @@ def test_starting_point_not_in_search_space(): print("PermissionError when deleting test/data/output/") -@pytest.mark.skipif(sys.platform == "win32", reason="do not run on windows") def test_points_to_evaluate(): from flaml import AutoML @@ -155,7 +165,6 @@ def test_points_to_evaluate(): # TODO: implement _test_zero_shot_model -@pytest.mark.skipif(sys.platform == "win32", reason="do not run on windows") def test_zero_shot_nomodel(): from flaml.default import preprocess_and_suggest_hyperparams diff --git a/test/spark/test_0sparkml.py b/test/spark/test_0sparkml.py index cb51b292f5..bc68637642 100644 --- a/test/spark/test_0sparkml.py +++ b/test/spark/test_0sparkml.py @@ -1,3 +1,4 @@ +import atexit import os import sys import warnings @@ -10,6 +11,7 @@ from flaml import AutoML from flaml.automl.data import auto_convert_dtypes_pandas, auto_convert_dtypes_spark, get_random_dataframe +from flaml.automl.spark import disable_spark_ansi_mode, restore_spark_ansi_mode from flaml.tune.spark.utils import check_spark warnings.simplefilter(action="ignore") @@ -29,7 +31,7 @@ .config( "spark.jars.packages", ( - "com.microsoft.azure:synapseml_2.12:1.0.4," + "com.microsoft.azure:synapseml_2.12:1.1.0," "org.apache.hadoop:hadoop-azure:3.3.5," "com.microsoft.azure:azure-storage:8.6.6," f"org.mlflow:mlflow-spark_2.12:{mlflow.__version__}" @@ -55,6 +57,9 @@ except ImportError: skip_spark = True +spark, ansi_conf, adjusted = disable_spark_ansi_mode() +atexit.register(restore_spark_ansi_mode, spark, ansi_conf, adjusted) + if sys.version_info >= (3, 11): skip_py311 = True else: @@ -64,6 +69,13 @@ def _test_spark_synapseml_lightgbm(spark=None, task="classification"): + # TODO: remove the estimator assignment once SynapseML supports spark 4+. + from flaml.automl.spark.utils import _spark_major_minor_version + + if _spark_major_minor_version[0] >= 4: + # skip synapseml lightgbm test for spark 4+ + return + if task == "classification": metric = "accuracy" X_train, y_train = skds.load_iris(return_X_y=True, as_frame=True) @@ -154,26 +166,31 @@ def test_spark_synapseml_rank(): def test_spark_input_df(): - df = ( - spark.read.format("csv") - .option("header", True) - .option("inferSchema", True) - .load("wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv") - ) + import pandas as pd + + file_url = "https://mmlspark.blob.core.windows.net/publicwasb/company_bankruptcy_prediction_data.csv" + df = pd.read_csv(file_url) + df = spark.createDataFrame(df) train, test = df.randomSplit([0.8, 0.2], seed=1) feature_cols = df.columns[1:] featurizer = VectorAssembler(inputCols=feature_cols, outputCol="features") train_data = featurizer.transform(train)["Bankrupt?", "features"] test_data = featurizer.transform(test)["Bankrupt?", "features"] automl = AutoML() + + # TODO: remove the estimator assignment once SynapseML supports spark 4+. + from flaml.automl.spark.utils import _spark_major_minor_version + + estimator_list = ["rf_spark"] if _spark_major_minor_version[0] >= 4 else None + settings = { "time_budget": 30, # total running time in seconds "metric": "roc_auc", - # "estimator_list": ["lgbm_spark"], # list of ML learners; we tune lightgbm in this example "task": "classification", # task type "log_file_name": "flaml_experiment.log", # flaml log file "seed": 7654321, # random seed "eval_method": "holdout", + "estimator_list": estimator_list, # TODO: remove once SynapseML supports spark 4+ } df = to_pandas_on_spark(to_pandas_on_spark(train_data).to_spark(index_col="index")) @@ -184,6 +201,9 @@ def test_spark_input_df(): **settings, ) + if estimator_list == ["rf_spark"]: + return + try: model = automl.model.estimator predictions = model.transform(test_data) diff --git a/test/spark/test_mlflow.py b/test/spark/test_mlflow.py index b61b978146..4eecea8ea3 100644 --- a/test/spark/test_mlflow.py +++ b/test/spark/test_mlflow.py @@ -1,3 +1,4 @@ +import atexit import importlib import os import sys @@ -13,6 +14,7 @@ from sklearn.model_selection import train_test_split import flaml +from flaml.automl.spark import disable_spark_ansi_mode, restore_spark_ansi_mode from flaml.automl.spark.utils import to_pandas_on_spark try: @@ -121,28 +123,39 @@ def _check_mlflow_logging(possible_num_runs, metric, is_parent_run, experiment_i @pytest.mark.skipif(skip_spark, reason="Spark is not installed. Skip all spark tests.") -def test_tune_autolog_parentrun_parallel(): - experiment_id = _test_tune(is_autolog=True, is_parent_run=True, is_parallel=True) - _check_mlflow_logging([4, 3], "r2", True, experiment_id) +def test_automl_nonsparkdata_noautolog_noparentrun(): + experiment_id = _test_automl_nonsparkdata(is_autolog=False, is_parent_run=False) + _check_mlflow_logging(0, "r2", False, experiment_id, is_automl=True) # no logging -def test_tune_autolog_parentrun_nonparallel(): - experiment_id = _test_tune(is_autolog=True, is_parent_run=True, is_parallel=False) - _check_mlflow_logging(3, "r2", True, experiment_id) +@pytest.mark.skipif(skip_spark, reason="Spark is not installed. Skip all spark tests.") +def test_automl_sparkdata_noautolog_noparentrun(): + experiment_id = _test_automl_sparkdata(is_autolog=False, is_parent_run=False) + _check_mlflow_logging(0, "mse", False, experiment_id, is_automl=True) # no logging @pytest.mark.skipif(skip_spark, reason="Spark is not installed. Skip all spark tests.") -def test_tune_autolog_noparentrun_parallel(): - experiment_id = _test_tune(is_autolog=True, is_parent_run=False, is_parallel=True) - _check_mlflow_logging([4, 3], "r2", False, experiment_id) +def test_tune_noautolog_noparentrun_parallel(): + experiment_id = _test_tune(is_autolog=False, is_parent_run=False, is_parallel=True) + _check_mlflow_logging(0, "r2", False, experiment_id) + + +def test_tune_noautolog_noparentrun_nonparallel(): + experiment_id = _test_tune(is_autolog=False, is_parent_run=False, is_parallel=False) + _check_mlflow_logging(3, "r2", False, experiment_id, skip_tags=True) @pytest.mark.skipif(skip_spark, reason="Spark is not installed. Skip all spark tests.") -def test_tune_noautolog_parentrun_parallel(): - experiment_id = _test_tune(is_autolog=False, is_parent_run=True, is_parallel=True) +def test_tune_autolog_parentrun_parallel(): + experiment_id = _test_tune(is_autolog=True, is_parent_run=True, is_parallel=True) _check_mlflow_logging([4, 3], "r2", True, experiment_id) +def test_tune_autolog_parentrun_nonparallel(): + experiment_id = _test_tune(is_autolog=True, is_parent_run=True, is_parallel=False) + _check_mlflow_logging(3, "r2", True, experiment_id) + + def test_tune_autolog_noparentrun_nonparallel(): experiment_id = _test_tune(is_autolog=True, is_parent_run=False, is_parallel=False) _check_mlflow_logging(3, "r2", False, experiment_id) @@ -154,17 +167,23 @@ def test_tune_noautolog_parentrun_nonparallel(): @pytest.mark.skipif(skip_spark, reason="Spark is not installed. Skip all spark tests.") -def test_tune_noautolog_noparentrun_parallel(): - experiment_id = _test_tune(is_autolog=False, is_parent_run=False, is_parallel=True) - _check_mlflow_logging(0, "r2", False, experiment_id) +def test_tune_autolog_noparentrun_parallel(): + experiment_id = _test_tune(is_autolog=True, is_parent_run=False, is_parallel=True) + _check_mlflow_logging([4, 3], "r2", False, experiment_id) -def test_tune_noautolog_noparentrun_nonparallel(): - experiment_id = _test_tune(is_autolog=False, is_parent_run=False, is_parallel=False) - _check_mlflow_logging(3, "r2", False, experiment_id, skip_tags=True) +@pytest.mark.skipif(skip_spark, reason="Spark is not installed. Skip all spark tests.") +def test_tune_noautolog_parentrun_parallel(): + experiment_id = _test_tune(is_autolog=False, is_parent_run=True, is_parallel=True) + _check_mlflow_logging([4, 3], "r2", True, experiment_id) def _test_automl_sparkdata(is_autolog, is_parent_run): + # TODO: remove the estimator assignment once SynapseML supports spark 4+. + from flaml.automl.spark.utils import _spark_major_minor_version + + estimator_list = ["rf_spark"] if _spark_major_minor_version[0] >= 4 else None + mlflow.end_run() mlflow_exp_name = f"test_mlflow_integration_{int(time.time())}" mlflow_experiment = mlflow.set_experiment(mlflow_exp_name) @@ -175,6 +194,9 @@ def _test_automl_sparkdata(is_autolog, is_parent_run): if is_parent_run: mlflow.start_run(run_name=f"automl_sparkdata_autolog_{is_autolog}") spark = pyspark.sql.SparkSession.builder.getOrCreate() + spark, ansi_conf, adjusted = disable_spark_ansi_mode() + atexit.register(restore_spark_ansi_mode, spark, ansi_conf, adjusted) + pd_df = load_diabetes(as_frame=True).frame df = spark.createDataFrame(pd_df) df = df.repartition(4).cache() @@ -193,6 +215,7 @@ def _test_automl_sparkdata(is_autolog, is_parent_run): "log_type": "all", "n_splits": 2, "model_history": True, + "estimator_list": estimator_list, } df = to_pandas_on_spark(to_pandas_on_spark(train_data).to_spark(index_col="index")) automl.fit( @@ -252,12 +275,6 @@ def test_automl_sparkdata_noautolog_parentrun(): _check_mlflow_logging(3, "mse", True, experiment_id, is_automl=True) -@pytest.mark.skipif(skip_spark, reason="Spark is not installed. Skip all spark tests.") -def test_automl_sparkdata_noautolog_noparentrun(): - experiment_id = _test_automl_sparkdata(is_autolog=False, is_parent_run=False) - _check_mlflow_logging(0, "mse", False, experiment_id, is_automl=True) # no logging - - @pytest.mark.skipif(skip_spark, reason="Spark is not installed. Skip all spark tests.") def test_automl_nonsparkdata_autolog_parentrun(): experiment_id = _test_automl_nonsparkdata(is_autolog=True, is_parent_run=True) @@ -276,12 +293,6 @@ def test_automl_nonsparkdata_noautolog_parentrun(): _check_mlflow_logging([4, 3], "r2", True, experiment_id, is_automl=True) -@pytest.mark.skipif(skip_spark, reason="Spark is not installed. Skip all spark tests.") -def test_automl_nonsparkdata_noautolog_noparentrun(): - experiment_id = _test_automl_nonsparkdata(is_autolog=False, is_parent_run=False) - _check_mlflow_logging(0, "r2", False, experiment_id, is_automl=True) # no logging - - @pytest.mark.skipif(skip_spark, reason="Spark is not installed. Skip all spark tests.") def test_exit_pyspark_autolog(): import pyspark @@ -319,6 +330,9 @@ def _init_spark_for_main(): "https://mmlspark.blob.core.windows.net/publicwasb/log_model_allowlist.txt", ) + spark, ansi_conf, adjusted = disable_spark_ansi_mode() + atexit.register(restore_spark_ansi_mode, spark, ansi_conf, adjusted) + if __name__ == "__main__": _init_spark_for_main() diff --git a/test/spark/test_performance.py b/test/spark/test_performance.py index 030fcae493..b3c76c9dfe 100644 --- a/test/spark/test_performance.py +++ b/test/spark/test_performance.py @@ -31,14 +31,14 @@ class OpenMLServerException(Exception): os.environ["FLAML_MAX_CONCURRENT"] = "2" -def run_automl(budget=3, dataset_format="dataframe", hpo_method=None): +def run_automl(budget=30, dataset_format="dataframe", hpo_method=None): import urllib3 from flaml.automl.data import load_openml_dataset performance_check_budget = 3600 if sys.platform == "darwin" or "nt" in os.name or "3.10" not in sys.version: - budget = 3 # revise the buget if the platform is not linux + python 3.10 + budget = 30 # revise the buget if the platform is not linux + python 3.10 if budget >= performance_check_budget: max_iter = 60 performance_check_budget = None @@ -91,6 +91,11 @@ def run_automl(budget=3, dataset_format="dataframe", hpo_method=None): print("Best ML leaner:", automl.best_estimator) print("Best hyperparmeter config:", automl.best_config) print(f"Best accuracy on validation data: {1 - automl.best_loss:.4g}") + if performance_check_budget is not None and automl.best_estimator is None: + # skip the performance check if no model is trained + # this happens sometimes in github actions ubuntu python 3.12 environment + print("Warning: no model is trained, skip performance check") + return print(f"Training duration of best run: {automl.best_config_train_time:.4g} s") print(automl.model.estimator) print(automl.best_config_per_estimator) diff --git a/test/spark/test_utils.py b/test/spark/test_utils.py index 1141972bee..8e787e9e83 100644 --- a/test/spark/test_utils.py +++ b/test/spark/test_utils.py @@ -1,3 +1,4 @@ +import atexit import os from functools import partial from timeit import timeit @@ -14,6 +15,7 @@ from pyspark.sql import SparkSession from flaml.automl.ml import sklearn_metric_loss_score + from flaml.automl.spark import disable_spark_ansi_mode, restore_spark_ansi_mode from flaml.automl.spark.metrics import spark_metric_loss_score from flaml.automl.spark.utils import ( iloc_pandas_on_spark, @@ -24,6 +26,7 @@ unique_value_first_index, ) from flaml.tune.spark.utils import ( + _spark_major_minor_version, check_spark, get_broadcast_data, get_n_cpus, @@ -35,10 +38,41 @@ except ImportError: print("Spark is not installed. Skip all spark tests.") skip_spark = True + _spark_major_minor_version = (0, 0) + pytestmark = [pytest.mark.skipif(skip_spark, reason="Spark is not installed. Skip all spark tests."), pytest.mark.spark] +@pytest.mark.skipif(_spark_major_minor_version[0] < 4, reason="Requires Spark 4.0+") +def test_to_pandas_on_spark_temp_override(): + import pyspark.pandas as ps + from pyspark.sql import Row + + from flaml.automl.spark.utils import to_pandas_on_spark + + spark_session = SparkSession.builder.getOrCreate() + spark, ansi_conf, adjusted = disable_spark_ansi_mode() + atexit.register(restore_spark_ansi_mode, spark, ansi_conf, adjusted) + + # Ensure we can toggle options + orig = ps.get_option("compute.fail_on_ansi_mode") + + try: + spark_session.conf.set("spark.sql.ansi.enabled", "true") + ps.set_option("compute.fail_on_ansi_mode", True) + + # create tiny spark df + sdf = spark_session.createDataFrame([Row(a=1, b=2)]) + # Should not raise as our function temporarily disables fail_on_ansi_mode + pds = to_pandas_on_spark(sdf) + assert "a" in pds.columns + finally: + # restore test environment + ps.set_option("compute.fail_on_ansi_mode", orig) + spark_session.conf.set("spark.sql.ansi.enabled", "false") + + def test_with_parameters_spark(): def train(config, data=None): if isinstance(data, pyspark.broadcast.Broadcast): diff --git a/test/tune/test_lexiflow.py b/test/tune/test_lexiflow.py index 45e4700a6d..7b12c74618 100644 --- a/test/tune/test_lexiflow.py +++ b/test/tune/test_lexiflow.py @@ -4,10 +4,17 @@ import numpy as np import pytest -import thop -import torch -import torch.nn as nn -import torch.nn.functional as F + +try: + import thop + import torch + import torch.nn as nn + import torch.nn.functional as F +except ImportError: + thop = None + torch = None + nn = None + F = None try: import torchvision @@ -16,6 +23,11 @@ from flaml import tune +if thop is None or torch is None or nn is None or F is None or torchvision is None: + pytest.skip( + "skipping test_lexiflow.py because torch, torchvision or thop is not installed.", allow_module_level=True + ) + DEVICE = torch.device("cpu") BATCHSIZE = 128 N_TRAIN_EXAMPLES = BATCHSIZE * 30 diff --git a/test/tune/test_tune.py b/test/tune/test_tune.py index 5f17272dc5..8fd98fce69 100644 --- a/test/tune/test_tune.py +++ b/test/tune/test_tune.py @@ -53,6 +53,11 @@ def _easy_objective(config): def test_nested_run(): + """ + nested tuning example: Tune -> AutoML -> MLflow autolog + mlflow logging is complicated in nested tuning. It's better to turn off mlflow autologging to avoid + potential issues in FLAML's mlflow_integration.adopt_children() function. + """ from flaml import AutoML, tune data, labels = sklearn.datasets.load_breast_cancer(return_X_y=True) diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 715727bd45..bfea688fdc 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -99,6 +99,12 @@ module.exports = { 'https://github.com/microsoft/FLAML/edit/main/website/', remarkPlugins: [math], rehypePlugins: [katex], + // Allow __init__.md and other underscore-prefixed markdown docs + exclude: [ + '**/_*.{js,jsx,ts,tsx}', + '**/*.test.{js,jsx,ts,tsx}', + '**/__tests__/**', + ], }, theme: { customCss: require.resolve('./src/css/custom.css'),