Skip to content

Commit ce8bb7e

Browse files
Copilotthinkall
Copilot
and
thinkall
authored
Fix AutoML tests when OpenML redirects fail (#1537)
* Handle OpenML redirect failures in tests Agent-Logs-Url: https://github.com/microsoft/FLAML/sessions/ef7e267b-7c76-41ae-bd4a-0a1b4ec5aefb Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com> * Guard empty metric constraint trials Agent-Logs-Url: https://github.com/microsoft/FLAML/sessions/ef7e267b-7c76-41ae-bd4a-0a1b4ec5aefb Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com> * Use robust synthetic fallback data Agent-Logs-Url: https://github.com/microsoft/FLAML/sessions/ef7e267b-7c76-41ae-bd4a-0a1b4ec5aefb Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com>
1 parent 768b948 commit ce8bb7e

4 files changed

Lines changed: 34 additions & 26 deletions

File tree

‎test/automl/test_constraints.py‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@ def test_metric_constraints():
3030
try:
3131
X, y = fetch_openml(name=dataset, return_X_y=True)
3232
except (ArffException, ValueError, URLError):
33-
from sklearn.datasets import load_wine
33+
from sklearn.datasets import make_classification
3434

35-
X, y = load_wine(return_X_y=True)
35+
X, y = make_classification(n_samples=1000, n_features=20, n_informative=10, random_state=42)
3636
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
3737
automl.fit(X_train=X_train, y_train=y_train, **automl_settings)
3838
print(automl.estimator_list)
@@ -59,7 +59,8 @@ def test_metric_constraints():
5959
metric_constraints=automl.metric_constraints,
6060
num_samples=5,
6161
)
62-
print(analysis.trials[-1])
62+
if analysis.trials:
63+
print(analysis.trials[-1])
6364

6465

6566
def custom_metric(
@@ -116,10 +117,10 @@ def test_metric_constraints_custom():
116117

117118
try:
118119
X, y = fetch_openml(name=dataset, return_X_y=True)
119-
except (ArffException, ValueError):
120-
from sklearn.datasets import load_wine
120+
except (ArffException, ValueError, URLError):
121+
from sklearn.datasets import make_classification
121122

122-
X, y = load_wine(return_X_y=True)
123+
X, y = make_classification(n_samples=1000, n_features=20, n_informative=10, random_state=42)
123124
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
124125
automl.fit(X_train=X_train, y_train=y_train, **automl_settings)
125126
print(automl.estimator_list)
@@ -156,7 +157,8 @@ def test_metric_constraints_custom():
156157
metric_constraints=automl.metric_constraints,
157158
num_samples=5,
158159
)
159-
print(analysis.trials[-1])
160+
if analysis.trials:
161+
print(analysis.trials[-1])
160162

161163

162164
if __name__ == "__main__":

‎test/automl/test_score.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
from urllib.error import URLError
2+
13
import pandas as pd
24
from sklearn.datasets import fetch_california_housing, fetch_openml
35

@@ -181,10 +183,10 @@ def test_rank(self):
181183
try:
182184
X, y = fetch_openml(name=dataset, return_X_y=True)
183185
y = y.cat.codes
184-
except (ArffException, ValueError):
185-
from sklearn.datasets import load_wine
186+
except (ArffException, ValueError, URLError):
187+
from sklearn.datasets import make_classification
186188

187-
X, y = load_wine(return_X_y=True)
189+
X, y = make_classification(n_samples=1000, n_features=20, n_informative=10, random_state=42)
188190

189191
import numpy as np
190192

‎test/automl/test_split.py‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
from urllib.error import URLError
2+
13
import numpy as np
24
import pandas as pd
35
from sklearn.datasets import fetch_openml, load_iris
@@ -26,10 +28,10 @@ def _test(split_type):
2628

2729
try:
2830
X, y = fetch_openml(name=dataset, return_X_y=True)
29-
except (ArffException, ValueError):
30-
from sklearn.datasets import load_wine
31+
except (ArffException, ValueError, URLError):
32+
from sklearn.datasets import make_classification
3133

32-
X, y = load_wine(return_X_y=True)
34+
X, y = make_classification(n_samples=1000, n_features=20, n_informative=10, random_state=42)
3335
if split_type != "time":
3436
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
3537
else:
@@ -55,10 +57,10 @@ def test_groups_for_classification_task():
5557

5658
try:
5759
X, y = fetch_openml(name=dataset, return_X_y=True)
58-
except (ArffException, ValueError):
59-
from sklearn.datasets import load_wine
60+
except (ArffException, ValueError, URLError):
61+
from sklearn.datasets import make_classification
6062

61-
X, y = load_wine(return_X_y=True)
63+
X, y = make_classification(n_samples=1000, n_features=20, n_informative=10, random_state=42)
6264

6365
automl = AutoML()
6466
automl_settings = {
@@ -193,10 +195,10 @@ def test_rank():
193195
try:
194196
X, y = fetch_openml(name=dataset, return_X_y=True)
195197
y = y.cat.codes
196-
except (ArffException, ValueError):
197-
from sklearn.datasets import load_wine
198+
except (ArffException, ValueError, URLError):
199+
from sklearn.datasets import make_classification
198200

199-
X, y = load_wine(return_X_y=True)
201+
X, y = make_classification(n_samples=1000, n_features=20, n_informative=10, random_state=42)
200202
import numpy as np
201203

202204
automl = AutoML()
@@ -230,10 +232,10 @@ def test_object():
230232

231233
try:
232234
X, y = fetch_openml(name=dataset, return_X_y=True)
233-
except (ArffException, ValueError):
234-
from sklearn.datasets import load_wine
235+
except (ArffException, ValueError, URLError):
236+
from sklearn.datasets import make_classification
235237

236-
X, y = load_wine(return_X_y=True)
238+
X, y = make_classification(n_samples=1000, n_features=20, n_informative=10, random_state=42)
237239

238240
import numpy as np
239241

‎test/automl/test_xgboost2d.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import unittest
2+
from urllib.error import URLError
23

34
from sklearn.datasets import fetch_openml
45
from sklearn.model_selection import train_test_split
@@ -45,10 +46,10 @@ def test_simple(method=None):
4546

4647
try:
4748
X, y = fetch_openml(name=dataset, return_X_y=True)
48-
except (ArffException, ValueError):
49-
from sklearn.datasets import load_wine
49+
except (ArffException, ValueError, URLError):
50+
from sklearn.datasets import make_classification
5051

51-
X, y = load_wine(return_X_y=True)
52+
X, y = make_classification(n_samples=1000, n_features=20, n_informative=10, random_state=42)
5253
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
5354
automl.fit(X_train=X_train, y_train=y_train, **automl_settings)
5455
print(automl.estimator_list)
@@ -80,7 +81,8 @@ def test_simple(method=None):
8081
metric_constraints=automl.metric_constraints,
8182
num_samples=5,
8283
)
83-
print(analysis.trials[-1])
84+
if analysis.trials:
85+
print(analysis.trials[-1])
8486

8587

8688
def test_optuna():

0 commit comments

Comments
 (0)