Merge pull request #49 from lambda-feedback/validation_fix

refactor: improve validation logic and update tests for epsilon trans…

Author: JohnnyWan1123

evaluation_function/schemas/__init__.py

@@ -18,7 +18,7 @@
     "Params",
     # Result
     "Result",
-    "ValidationResult"
+    "ValidationResult",
     "ValidationError",
     "ElementHighlight",
     "ErrorCode",

evaluation_function/test/test_correction.py

@@ -214,7 +214,19 @@ def test_non_minimal_fsa_fails_when_required(self, equivalent_dfa):
 class TestEpsilonTransitionCorrection:
     """Test the full correction pipeline with ε-NFA inputs."""
 
-    def test_epsilon_nfa_vs_equivalent_dfa_correct(self):
+    @pytest.fixture
+    def nfa_params(self):
+        """Params that allow NFA/ε-NFA student submissions."""
+        return Params(
+            expected_type="any",
+            check_completeness=False,
+            check_minimality=False,
+            evaluation_mode="lenient",
+            highlight_errors=True,
+            feedback_verbosity="detailed",
+        )
+
+    def test_epsilon_nfa_vs_equivalent_dfa_correct(self, nfa_params):
         """ε-NFA student answer equivalent to DFA expected should be correct."""
         # ε-NFA accepts exactly "a": q0 --ε--> q1 --a--> q2
         student_enfa = make_fsa(
@@ -237,11 +249,11 @@ def test_epsilon_nfa_vs_equivalent_dfa_correct(self):
             initial="s0",
             accept=["s1"],
         )
-        result = analyze_fsa_correction(student_enfa, expected_dfa)
+        result = analyze_fsa_correction(student_enfa, expected_dfa, nfa_params)
         assert isinstance(result, Result)
         assert result.is_correct is True
 
-    def test_epsilon_nfa_vs_different_dfa_incorrect(self):
+    def test_epsilon_nfa_vs_different_dfa_incorrect(self, nfa_params):
         """ε-NFA accepting 'a' vs DFA accepting 'b' should be incorrect."""
         student_enfa = make_fsa(
             states=["q0", "q1", "q2"],
@@ -262,13 +274,13 @@ def test_epsilon_nfa_vs_different_dfa_incorrect(self):
             initial="s0",
             accept=["s1"],
         )
-        result = analyze_fsa_correction(student_enfa, expected_dfa)
+        result = analyze_fsa_correction(student_enfa, expected_dfa, nfa_params)
         assert isinstance(result, Result)
         assert result.is_correct is False
         assert result.fsa_feedback is not None
         assert len(result.fsa_feedback.errors) > 0
 
-    def test_multi_epsilon_nfa_vs_dfa_correct(self):
+    def test_multi_epsilon_nfa_vs_dfa_correct(self, nfa_params):
         """ε-NFA for (a|b) with branching epsilons should match equivalent DFA."""
         student_enfa = make_fsa(
             states=["q0", "q1", "q2", "q3"],
@@ -292,11 +304,11 @@ def test_multi_epsilon_nfa_vs_dfa_correct(self):
             initial="s0",
             accept=["s1"],
         )
-        result = analyze_fsa_correction(student_enfa, expected_dfa)
+        result = analyze_fsa_correction(student_enfa, expected_dfa, nfa_params)
         assert isinstance(result, Result)
         assert result.is_correct is True
 
-    def test_epsilon_nfa_structural_info_reports_nondeterministic(self):
+    def test_epsilon_nfa_structural_info_reports_nondeterministic(self, nfa_params):
         """ε-NFA should have structural info reporting non-deterministic."""
         student_enfa = make_fsa(
             states=["q0", "q1", "q2"],
@@ -317,7 +329,7 @@ def test_epsilon_nfa_structural_info_reports_nondeterministic(self):
             initial="s0",
             accept=["s1"],
         )
-        result = analyze_fsa_correction(student_enfa, expected_dfa)
+        result = analyze_fsa_correction(student_enfa, expected_dfa, nfa_params)
         assert result.fsa_feedback is not None
         assert result.fsa_feedback.structural is not None
         assert result.fsa_feedback.structural.is_deterministic is False

evaluation_function/test/test_validation.py

@@ -314,7 +314,7 @@ def test_isomorphic_dfas(self):
             initial="s0",
             accept=["s1"],
         )
-        assert are_isomorphic(fsa_user, fsa_sol) == []
+        assert are_isomorphic(fsa_user, fsa_sol).ok
 
 
 class TestEpsilonTransitions:
@@ -332,7 +332,7 @@ def test_valid_fsa_with_epsilon_unicode(self):
             initial="q0",
             accept=["q2"],
         )
-        assert is_valid_fsa(fsa) == []
+        assert is_valid_fsa(fsa).ok
 
     def test_valid_fsa_with_epsilon_string(self):
         """ε-NFA with 'epsilon' string should pass structural validation."""
@@ -346,7 +346,7 @@ def test_valid_fsa_with_epsilon_string(self):
             initial="q0",
             accept=["q2"],
         )
-        assert is_valid_fsa(fsa) == []
+        assert is_valid_fsa(fsa).ok
 
     def test_valid_fsa_with_empty_string_epsilon(self):
         """ε-NFA with empty string epsilon should pass structural validation."""
@@ -360,7 +360,7 @@ def test_valid_fsa_with_empty_string_epsilon(self):
             initial="q0",
             accept=["q2"],
         )
-        assert is_valid_fsa(fsa) == []
+        assert is_valid_fsa(fsa).ok
 
     def test_epsilon_nfa_is_not_deterministic(self):
         """ε-NFA should be flagged as non-deterministic."""
@@ -373,9 +373,9 @@ def test_epsilon_nfa_is_not_deterministic(self):
             initial="q0",
             accept=["q1"],
         )
-        errors = is_deterministic(fsa)
-        assert len(errors) > 0
-        assert ErrorCode.NOT_DETERMINISTIC in [e.code for e in errors]
+        result = is_deterministic(fsa)
+        assert not result.ok
+        assert ErrorCode.NOT_DETERMINISTIC in [e.code for e in result.errors]
 
     def test_accepts_string_via_epsilon_closure(self):
         """ε-NFA should accept 'a' by following q0 --ε--> q1 --a--> q2."""
@@ -389,7 +389,7 @@ def test_accepts_string_via_epsilon_closure(self):
             initial="q0",
             accept=["q2"],
         )
-        assert accepts_string(fsa, "a") == []
+        assert accepts_string(fsa, "a").ok
 
     def test_rejects_string_with_epsilon_nfa(self):
         """ε-NFA that accepts 'a' should reject empty string."""
@@ -403,8 +403,8 @@ def test_rejects_string_with_epsilon_nfa(self):
             initial="q0",
             accept=["q2"],
         )
-        errors = accepts_string(fsa, "")
-        assert len(errors) > 0
+        result = accepts_string(fsa, "")
+        assert not result.ok
 
     def test_accepts_empty_string_via_epsilon(self):
         """ε-NFA should accept empty string when initial reaches accept via ε."""
@@ -417,7 +417,7 @@ def test_accepts_empty_string_via_epsilon(self):
             initial="q0",
             accept=["q1"],
         )
-        assert accepts_string(fsa, "") == []
+        assert accepts_string(fsa, "").ok
 
     def test_epsilon_nfa_equivalent_to_dfa(self):
         """ε-NFA and DFA accepting the same language should be equivalent."""
@@ -440,7 +440,7 @@ def test_epsilon_nfa_equivalent_to_dfa(self):
             initial="s0",
             accept=["s1"],
         )
-        assert fsas_accept_same_language(enfa, dfa) == []
+        assert fsas_accept_same_language(enfa, dfa).ok
 
     def test_epsilon_nfa_not_equivalent_to_different_dfa(self):
         """ε-NFA and DFA accepting different languages should not be equivalent."""
@@ -463,8 +463,8 @@ def test_epsilon_nfa_not_equivalent_to_different_dfa(self):
             initial="s0",
             accept=["s1"],
         )
-        errors = fsas_accept_same_language(enfa, dfa)
-        assert len(errors) > 0
+        result = fsas_accept_same_language(enfa, dfa)
+        assert not result.ok
 
     def test_multi_epsilon_nfa_equivalent_to_dfa(self):
         """ε-NFA for (a|b) with branching epsilons should match equivalent DFA."""
@@ -491,7 +491,7 @@ def test_multi_epsilon_nfa_equivalent_to_dfa(self):
             initial="s0",
             accept=["s1"],
         )
-        assert fsas_accept_same_language(enfa, dfa) == []
+        assert fsas_accept_same_language(enfa, dfa).ok
 
 
 if __name__ == "__main__":