Skip to content

Commit 4a0be56

Browse files
biefanbiefan
committed
Fix unit test type check maintenance
1 parent d5b6cf0 commit 4a0be56

2 files changed

Lines changed: 56 additions & 39 deletions

File tree

‎pyproject.toml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,23 @@ include = ["pyrit/prompt_target/hugging_face/**"]
209209
[tool.ty.overrides.rules]
210210
invalid-argument-type = "ignore"
211211

212+
# Unit tests intentionally exercise runtime validation failures, mutable test doubles,
213+
# and optional fields after fixtures have populated them. Keep library code strict while
214+
# avoiding hundreds of per-assertion ignores in tests.
215+
[[tool.ty.overrides]]
216+
include = ["tests/unit/**"]
217+
[tool.ty.overrides.rules]
218+
call-non-callable = "ignore"
219+
invalid-assignment = "ignore"
220+
invalid-await = "ignore"
221+
invalid-parameter-default = "ignore"
222+
invalid-type-arguments = "ignore"
223+
invalid-yield = "ignore"
224+
missing-argument = "ignore"
225+
not-subscriptable = "ignore"
226+
unknown-argument = "ignore"
227+
unsupported-operator = "ignore"
228+
212229
[tool.uv]
213230
constraint-dependencies = [
214231
"aiohttp>=3.13.4",

‎tests/unit/score/test_scorer_eval_csv_schema.py‎

Lines changed: 39 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,28 @@
4141
VALID_DATA_TYPES = set(PromptDataType.__args__) # type: ignore[attr-defined]
4242

4343
# Collect all CSV paths once for cross-cutting tests
44-
ALL_CSV_FILES = (
45-
list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv"))
46-
+ list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv"))
47-
+ list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv"))
48-
)
44+
OBJECTIVE_CSV_FILES = list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv"))
45+
HARM_CSV_FILES = list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv"))
46+
REFUSAL_CSV_FILES = list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv"))
47+
ALL_CSV_FILES = OBJECTIVE_CSV_FILES + HARM_CSV_FILES + REFUSAL_CSV_FILES
48+
49+
50+
@pytest.fixture(scope="class")
51+
def objective_csv_files() -> list[Path]:
52+
"""Get all CSV files in the objective scorer evals directory."""
53+
return OBJECTIVE_CSV_FILES
54+
55+
56+
@pytest.fixture(scope="class")
57+
def harm_csv_files() -> list[Path]:
58+
"""Get all CSV files in the harm scorer evals directory."""
59+
return HARM_CSV_FILES
60+
61+
62+
@pytest.fixture(scope="class")
63+
def refusal_csv_files() -> list[Path]:
64+
"""Get all CSV files in the refusal scorer evals directory."""
65+
return REFUSAL_CSV_FILES
4966

5067

5168
def _skip_comment_lines(f) -> None:
@@ -89,18 +106,13 @@ def _read_csv_as_dataframe(csv_file: Path) -> pd.DataFrame:
89106
class TestObjectiveScorerEvalCSVSchema:
90107
"""Test that all objective scorer evaluation CSVs have the correct schema."""
91108

92-
@pytest.fixture(scope="class")
93-
def objective_csv_files(self) -> list[Path]:
94-
"""Get all CSV files in the objective scorer evals directory."""
95-
return list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv"))
96-
97109
def test_objective_csv_files_exist(self, objective_csv_files: list[Path]) -> None:
98110
"""Verify that objective CSV files exist."""
99111
assert len(objective_csv_files) > 0, "No objective CSV files found"
100112

101113
@pytest.mark.parametrize(
102114
"csv_file",
103-
list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv")),
115+
OBJECTIVE_CSV_FILES,
104116
ids=lambda p: p.name,
105117
)
106118
def test_objective_csv_has_required_columns(self, csv_file: Path) -> None:
@@ -133,7 +145,7 @@ def test_objective_csv_has_required_columns(self, csv_file: Path) -> None:
133145

134146
@pytest.mark.parametrize(
135147
"csv_file",
136-
list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv")),
148+
OBJECTIVE_CSV_FILES,
137149
ids=lambda p: p.name,
138150
)
139151
def test_objective_csv_column_names_exact(self, csv_file: Path) -> None:
@@ -164,7 +176,7 @@ def test_objective_csv_column_names_exact(self, csv_file: Path) -> None:
164176

165177
@pytest.mark.parametrize(
166178
"csv_file",
167-
list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv")),
179+
OBJECTIVE_CSV_FILES,
168180
ids=lambda p: p.name,
169181
)
170182
def test_objective_csv_scores_are_binary(self, csv_file: Path) -> None:
@@ -187,7 +199,7 @@ def test_objective_csv_scores_are_binary(self, csv_file: Path) -> None:
187199

188200
@pytest.mark.parametrize(
189201
"csv_file",
190-
list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv")),
202+
OBJECTIVE_CSV_FILES,
191203
ids=lambda p: p.name,
192204
)
193205
def test_objective_csv_loads_via_from_csv(self, csv_file: Path) -> None:
@@ -213,18 +225,13 @@ def test_objective_csv_loads_via_from_csv(self, csv_file: Path) -> None:
213225
class TestHarmScorerEvalCSVSchema:
214226
"""Test that all harm scorer evaluation CSVs have the correct schema."""
215227

216-
@pytest.fixture(scope="class")
217-
def harm_csv_files(self) -> list[Path]:
218-
"""Get all CSV files in the harm scorer evals directory."""
219-
return list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv"))
220-
221228
def test_harm_csv_files_exist(self, harm_csv_files: list[Path]) -> None:
222229
"""Verify that harm CSV files exist."""
223230
assert len(harm_csv_files) > 0, "No harm CSV files found"
224231

225232
@pytest.mark.parametrize(
226233
"csv_file",
227-
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
234+
HARM_CSV_FILES,
228235
ids=lambda p: p.name,
229236
)
230237
def test_harm_csv_has_required_columns(self, csv_file: Path) -> None:
@@ -269,7 +276,7 @@ def test_harm_csv_has_required_columns(self, csv_file: Path) -> None:
269276

270277
@pytest.mark.parametrize(
271278
"csv_file",
272-
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
279+
HARM_CSV_FILES,
273280
ids=lambda p: p.name,
274281
)
275282
def test_harm_csv_has_human_score_columns(self, csv_file: Path) -> None:
@@ -296,7 +303,7 @@ def test_harm_csv_has_human_score_columns(self, csv_file: Path) -> None:
296303

297304
@pytest.mark.parametrize(
298305
"csv_file",
299-
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
306+
HARM_CSV_FILES,
300307
ids=lambda p: p.name,
301308
)
302309
def test_harm_csv_has_harm_definition(self, csv_file: Path) -> None:
@@ -317,7 +324,7 @@ def test_harm_csv_has_harm_definition(self, csv_file: Path) -> None:
317324

318325
@pytest.mark.parametrize(
319326
"csv_file",
320-
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
327+
HARM_CSV_FILES,
321328
ids=lambda p: p.name,
322329
)
323330
def test_harm_definition_file_exists_and_is_valid(self, csv_file: Path) -> None:
@@ -365,7 +372,7 @@ def test_harm_definition_file_exists_and_is_valid(self, csv_file: Path) -> None:
365372

366373
@pytest.mark.parametrize(
367374
"csv_file",
368-
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
375+
HARM_CSV_FILES,
369376
ids=lambda p: p.name,
370377
)
371378
def test_harm_csv_scores_in_valid_range(self, csv_file: Path) -> None:
@@ -387,7 +394,7 @@ def test_harm_csv_scores_in_valid_range(self, csv_file: Path) -> None:
387394

388395
@pytest.mark.parametrize(
389396
"csv_file",
390-
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
397+
HARM_CSV_FILES,
391398
ids=lambda p: p.name,
392399
)
393400
def test_harm_csv_single_harm_category(self, csv_file: Path) -> None:
@@ -409,7 +416,7 @@ def test_harm_csv_single_harm_category(self, csv_file: Path) -> None:
409416

410417
@pytest.mark.parametrize(
411418
"csv_file",
412-
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
419+
HARM_CSV_FILES,
413420
ids=lambda p: p.name,
414421
)
415422
def test_harm_csv_definition_version_matches_yaml(self, csv_file: Path) -> None:
@@ -459,7 +466,7 @@ def test_harm_csv_definition_version_matches_yaml(self, csv_file: Path) -> None:
459466

460467
@pytest.mark.parametrize(
461468
"csv_file",
462-
list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv")),
469+
HARM_CSV_FILES,
463470
ids=lambda p: p.name,
464471
)
465472
def test_harm_csv_loads_via_from_csv(self, csv_file: Path) -> None:
@@ -485,18 +492,13 @@ def test_harm_csv_loads_via_from_csv(self, csv_file: Path) -> None:
485492
class TestRefusalScorerEvalCSVSchema:
486493
"""Test that all refusal scorer evaluation CSVs have the correct schema."""
487494

488-
@pytest.fixture(scope="class")
489-
def refusal_csv_files(self) -> list[Path]:
490-
"""Get all CSV files in the refusal scorer evals directory."""
491-
return list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv"))
492-
493495
def test_refusal_csv_files_exist(self, refusal_csv_files: list[Path]) -> None:
494496
"""Verify that refusal CSV files exist."""
495497
assert len(refusal_csv_files) > 0, "No refusal CSV files found"
496498

497499
@pytest.mark.parametrize(
498500
"csv_file",
499-
list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv")),
501+
REFUSAL_CSV_FILES,
500502
ids=lambda p: p.name,
501503
)
502504
def test_refusal_csv_has_required_columns(self, csv_file: Path) -> None:
@@ -529,7 +531,7 @@ def test_refusal_csv_has_required_columns(self, csv_file: Path) -> None:
529531

530532
@pytest.mark.parametrize(
531533
"csv_file",
532-
list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv")),
534+
REFUSAL_CSV_FILES,
533535
ids=lambda p: p.name,
534536
)
535537
def test_refusal_csv_column_names_exact(self, csv_file: Path) -> None:
@@ -558,7 +560,7 @@ def test_refusal_csv_column_names_exact(self, csv_file: Path) -> None:
558560

559561
@pytest.mark.parametrize(
560562
"csv_file",
561-
list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv")),
563+
REFUSAL_CSV_FILES,
562564
ids=lambda p: p.name,
563565
)
564566
def test_refusal_csv_scores_are_binary(self, csv_file: Path) -> None:
@@ -581,7 +583,7 @@ def test_refusal_csv_scores_are_binary(self, csv_file: Path) -> None:
581583

582584
@pytest.mark.parametrize(
583585
"csv_file",
584-
list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv")),
586+
REFUSAL_CSV_FILES,
585587
ids=lambda p: p.name,
586588
)
587589
def test_refusal_csv_loads_via_from_csv(self, csv_file: Path) -> None:
@@ -614,9 +616,7 @@ class TestCSVVersionMetadata:
614616

615617
@pytest.mark.parametrize(
616618
"csv_file",
617-
list(Path(SCORER_EVALS_OBJECTIVE_PATH).glob("*.csv"))
618-
+ list(Path(SCORER_EVALS_HARM_PATH).glob("*.csv"))
619-
+ list(Path(SCORER_EVALS_REFUSAL_SCORER_PATH).glob("*.csv")),
619+
ALL_CSV_FILES,
620620
ids=lambda p: f"{p.parent.name}/{p.name}",
621621
)
622622
def test_csv_has_dataset_version_line(self, csv_file: Path) -> None:

0 commit comments

Comments
 (0)