Skip to content

Commit 0c2c63f

Browse files
ashwin6-devashwin6-dev
committed
feat: returning any exception as result for debugging
1 parent 08ccb1a commit 0c2c63f

1 file changed

Lines changed: 108 additions & 100 deletions

File tree

‎evaluation_function/evaluation.py‎

Lines changed: 108 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -39,139 +39,147 @@ def evaluation_function(
3939
to output the evaluation response.
4040
"""
4141

42-
if not isinstance(answer, str):
43-
return Result(
44-
is_correct=False,
45-
feedback_items=[("incorrect input", "missing answer object")]
46-
)
4742

48-
# If response is a string, parse it as JSON
49-
if isinstance(response, str):
50-
try:
51-
response = json.loads(response)
52-
except json.JSONDecodeError as e:
43+
try:
44+
if not isinstance(answer, str):
5345
return Result(
5446
is_correct=False,
55-
feedback_items=[("incorrect input", f"response is not valid JSON: {str(e)}")]
47+
feedback_items=[("incorrect input", "missing answer object")]
5648
)
5749

58-
if not isinstance(response, dict):
59-
return Result(
60-
is_correct=False,
61-
feedback_items=[("incorrect input", "missing response object")]
62-
)
63-
64-
65-
response_formula = response.get("formula", None)
66-
if not isinstance(response_formula, str):
67-
return Result(
68-
is_correct=False,
69-
feedback_items=[("incorrect input", "response must be type String")]
70-
)
71-
72-
# parse response_formula into Formula
73-
try:
74-
formula = formula_parser(response_formula)
75-
76-
except BuildError as e:
77-
return Result(
78-
is_correct=False,
79-
feedback_items=[(BuildError, str(e))]
80-
)
81-
except ValueError as e:
82-
return Result(
83-
is_correct=False,
84-
feedback_items=[(ValueError, str(e))]
85-
)
86-
87-
88-
89-
# check if input is a truth table
90-
truth_table = response.get("truthTable", None)
91-
if truth_table is not None and isinstance(truth_table, dict):
50+
# If response is a string, parse it as JSON
51+
if isinstance(response, str):
52+
try:
53+
response = json.loads(response)
54+
except json.JSONDecodeError as e:
55+
return Result(
56+
is_correct=False,
57+
feedback_items=[("incorrect input", f"response is not valid JSON: {str(e)}")]
58+
)
59+
60+
if not isinstance(response, dict):
61+
return Result(
62+
is_correct=False,
63+
feedback_items=[("incorrect input", "missing response object")]
64+
)
9265

93-
variables = truth_table.get("variables", [])
94-
cells = truth_table.get("cells", [])
95-
96-
if not isinstance(variables, list) or not isinstance(cells, list):
66+
67+
response_formula = response.get("formula", None)
68+
if not isinstance(response_formula, str):
9769
return Result(
9870
is_correct=False,
99-
feedback_items=[("incorrect input", "truthTable must contain 'variables' and 'cells' arrays")]
71+
feedback_items=[("incorrect input", "response must be type String")]
10072
)
10173

102-
# tokenise answer
74+
# parse response_formula into Formula
10375
try:
104-
answer_formula = formula_parser(answer)
76+
formula = formula_parser(response_formula)
10577

10678
except BuildError as e:
10779
return Result(
10880
is_correct=False,
109-
feedback_items=[("BuildError", str(e))]
81+
feedback_items=[(BuildError, str(e))]
11082
)
11183
except ValueError as e:
11284
return Result(
11385
is_correct=False,
114-
feedback_items=[("ValueError", str(e))]
86+
feedback_items=[(ValueError, str(e))]
11587
)
11688

117-
num_atoms = len(_extract_atoms(answer_formula))
118-
119-
# Evaluate the truth table
120-
truth_table_result = evaluate_truth_table(variables, cells, num_atoms)
121-
if not truth_table_result.is_correct:
122-
return truth_table_result
123-
12489

125-
126-
# check only one of "equivilance", "tautology", "satisfiability" is selected
127-
128-
equivalence = params.get("equivalence", False)
129-
tautology = params.get("tautology", False)
130-
satisfiability = params.get("satisfiability", False)
13190

91+
# check if input is a truth table
92+
truth_table = response.get("truthTable", None)
93+
if truth_table is not None and isinstance(truth_table, dict):
94+
95+
variables = truth_table.get("variables", [])
96+
cells = truth_table.get("cells", [])
97+
98+
if not isinstance(variables, list) or not isinstance(cells, list):
99+
return Result(
100+
is_correct=False,
101+
feedback_items=[("incorrect input", "truthTable must contain 'variables' and 'cells' arrays")]
102+
)
103+
104+
# tokenise answer
105+
try:
106+
answer_formula = formula_parser(answer)
107+
108+
except BuildError as e:
109+
return Result(
110+
is_correct=False,
111+
feedback_items=[("BuildError", str(e))]
112+
)
113+
except ValueError as e:
114+
return Result(
115+
is_correct=False,
116+
feedback_items=[("ValueError", str(e))]
117+
)
118+
119+
num_atoms = len(_extract_atoms(answer_formula))
120+
121+
# Evaluate the truth table
122+
truth_table_result = evaluate_truth_table(variables, cells, num_atoms)
123+
if not truth_table_result.is_correct:
124+
return truth_table_result
132125

133-
#check that 1 and only 1 param is selected
134-
num_selected = sum([equivalence, tautology, satisfiability])
135126

136-
if num_selected == 0:
137-
return Result(
138-
is_correct=False,
139-
feedback_items=[("invalid param", "please select a param")]
140-
)
141-
elif num_selected > 1:
142-
return Result(
143-
is_correct=False,
144-
feedback_items=[("invalid param", "please only select 1 param")]
145-
)
146-
147-
148-
feedback = None
149-
is_correct = False
127+
128+
# check only one of "equivilance", "tautology", "satisfiability" is selected
129+
130+
equivalence = params.get("equivalence", False)
131+
tautology = params.get("tautology", False)
132+
satisfiability = params.get("satisfiability", False)
150133

151134

152-
if equivalence:
135+
#check that 1 and only 1 param is selected
136+
num_selected = sum([equivalence, tautology, satisfiability])
153137

154-
# tokenise answer
155-
try:
156-
answer_formula = formula_parser(answer)
157-
158-
except BuildError as e:
138+
if num_selected == 0:
159139
return Result(
160140
is_correct=False,
161-
feedback_items=[(BuildError, str(e))]
141+
feedback_items=[("invalid param", "please select a param")]
162142
)
163-
except ValueError as e:
143+
elif num_selected > 1:
164144
return Result(
165145
is_correct=False,
166-
feedback_items=[(ValueError, str(e))]
146+
feedback_items=[("invalid param", "please only select 1 param")]
167147
)
148+
168149

169-
is_correct = EquivalenceEvaluator(formula, answer_formula).evaluate()
150+
feedback = None
151+
is_correct = False
152+
153+
154+
if equivalence:
155+
156+
# tokenise answer
157+
try:
158+
answer_formula = formula_parser(answer)
159+
160+
except BuildError as e:
161+
return Result(
162+
is_correct=False,
163+
feedback_items=[(BuildError, str(e))]
164+
)
165+
except ValueError as e:
166+
return Result(
167+
is_correct=False,
168+
feedback_items=[(ValueError, str(e))]
169+
)
170+
171+
is_correct = EquivalenceEvaluator(formula, answer_formula).evaluate()
172+
173+
174+
elif tautology:
175+
is_correct = TautologyEvaluator(formula).evaluate()
176+
elif satisfiability:
177+
is_correct = SatisfiabilityEvaluator(formula).evaluate()
178+
179+
return Result(is_correct=is_correct)
170180

171-
172-
elif tautology:
173-
is_correct = TautologyEvaluator(formula).evaluate()
174-
elif satisfiability:
175-
is_correct = SatisfiabilityEvaluator(formula).evaluate()
176-
177-
return Result(is_correct=is_correct)
181+
except Exception as e:
182+
return Result(
183+
is_correct=False,
184+
feedback_items=[("Error", str(e))]
185+
)

0 commit comments

Comments
 (0)