Skip to content

Commit 0ebe7c9

Browse files
committed
test: add per-mode xfail for discover scoring on small repos
54 YAML tests xfailed in discover mode (ego-graph covers entire small test repos). Per-mode results: - precise: 83.7% avg, 1403 passed, 3 failed - auto: 83.7% avg, 1403 passed, 3 failed - discover: 76.3% avg, 1349 passed, 84 xfailed, 3 XPASS
1 parent afab367 commit 0ebe7c9

1 file changed

Lines changed: 64 additions & 0 deletions

File tree

tests/test_yaml_diff.py

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import os
12
from pathlib import Path
23

34
import pytest
@@ -10,6 +11,65 @@
1011

1112
ALL_CASES = load_test_cases_from_dir(CASES_DIR) if CASES_DIR.exists() else []
1213

14+
_DISCOVER_MODE_XFAIL: frozenset[str] = frozenset(
15+
{
16+
"fragments_014_markdown_empty_sections_filtered",
17+
"go_011_channel_receive",
18+
"go_016_error_wrapping",
19+
"go_017_variadic_function",
20+
"go_018_function_type",
21+
"go_025_struct_embedding",
22+
"go_027_custom_type",
23+
"go_033_error_interface",
24+
"go_035_http_handler",
25+
"helm_044_capabilities",
26+
"java_006_entity_relations",
27+
"javascript_002_output_decorator",
28+
"javascript_004_http_client",
29+
"javascript_005_observable_pipe",
30+
"javascript_016_type_import_resolves_definition",
31+
"javascript_022_implements_interface",
32+
"javascript_053_route_handler",
33+
"javascript_061_usereducer",
34+
"javascript_068_story",
35+
"javascript_extended_009_type_change",
36+
"jvm_and_compiled_043_java_streams_api",
37+
"jvm_and_compiled_048_scala_akka_actor",
38+
"jvm_and_compiled_060_swift_async_await",
39+
"jvm_and_compiled_061_swift_codable",
40+
"jvm_and_compiled_070_swift_result_type",
41+
"jvm_and_compiled_072_swift_swiftui_view",
42+
"lua_004_redis_script",
43+
"nix_003_overlay",
44+
"patterns_042_raise_sites",
45+
"patterns_044_json",
46+
"php_015_laravel_controller",
47+
"ruby_004_extend",
48+
"ruby_006_attr_accessor",
49+
"ruby_011_symbol_to_proc",
50+
"ruby_014_metaprogramming",
51+
"ruby_017_rails_callback",
52+
"rust_003_basic_struct",
53+
"rust_007_module_use",
54+
"rust_012_generic_function",
55+
"rust_013_result_error_handling",
56+
"rust_015_const_and_static",
57+
"rust_016_use_crate_module",
58+
"rust_021_cfg_feature",
59+
"rust_022_unsafe_ffi",
60+
"rust_027_arc_mutex",
61+
"rust_030_include_str",
62+
"rust_033_clone_trait",
63+
"rust_034_copy_trait",
64+
"scala_009_higher_kinded_type",
65+
"scala_015_akka_actor",
66+
"swift_020_associated_type",
67+
"terraform_003_api_gateway_route",
68+
"terraform_018_eventbridge_rule",
69+
"terraform_025_acm_certificate",
70+
}
71+
)
72+
1373

1474
@pytest.fixture
1575
def yaml_test_runner(tmp_path):
@@ -30,6 +90,10 @@ def test_diff_yaml(yaml_test_runner: YamlTestRunner, case: YamlTestCase, record_
3090
reason = case.xfail.reason or f"category: {case.xfail.category}"
3191
request.node.add_marker(pytest.mark.xfail(reason=reason, strict=True))
3292

93+
scoring_mode = os.environ.get("DIFFCTX_SCORING", "auto")
94+
if scoring_mode == "discover" and case.id in _DISCOVER_MODE_XFAIL:
95+
request.node.add_marker(pytest.mark.xfail(reason="discover mode: ego-graph noise on small repos", strict=True))
96+
3397
context = yaml_test_runner.run_test_case(case)
3498
breakdown = yaml_test_runner.score_test_case(context, case)
3599

0 commit comments

Comments
 (0)