|
9 | 9 | \multicolumn{5}{l}{\textit{Syntactic correctness}} \\ |
10 | 10 | \quad\textcolor{gray}{\small In-dist (Verified)} & \textcolor{gray}{\small 0.698} & \textcolor{gray}{\small 0.669} & \textcolor{gray}{\small 0.679} & \textcolor{gray}{\small 0.626} \\ |
11 | 11 | \quad\textcolor{gray}{\small In-dist (Pro)} & \textcolor{gray}{\small 0.780} & \textcolor{gray}{\small 0.746} & \textcolor{gray}{\small 0.757} & \textcolor{gray}{\small 0.756} \\ |
12 | | -\quad Verified $\rightarrow$ Pro & 0.513\,{\scriptsize −0.266} & 0.505\,{\scriptsize −0.241} & 0.537\,{\scriptsize −0.220} & 0.520\,{\scriptsize −0.236} \\ |
13 | | -\quad Pro $\rightarrow$ Verified & 0.508\,{\scriptsize −0.190} & 0.561\,{\scriptsize −0.108} & 0.524\,{\scriptsize −0.155} & 0.488\,{\scriptsize −0.138} \\ |
| 12 | +\quad Verified $\rightarrow$ Pro & 0.513\,{\scriptsize $-0.266$} & 0.505\,{\scriptsize $-0.241$} & 0.537\,{\scriptsize $-0.220$} & 0.520\,{\scriptsize $-0.236$} \\ |
| 13 | +\quad Pro $\rightarrow$ Verified & 0.508\,{\scriptsize $-0.190$} & 0.561\,{\scriptsize $-0.108$} & 0.524\,{\scriptsize $-0.155$} & 0.488\,{\scriptsize $-0.138$} \\ |
14 | 14 | \midrule |
15 | 15 | \multicolumn{5}{l}{\textit{Functional correctness}} \\ |
16 | 16 | \quad\textcolor{gray}{\small In-dist (Verified)} & \textcolor{gray}{\small 0.775} & \textcolor{gray}{\small 0.791} & \textcolor{gray}{\small 0.769} & \textcolor{gray}{\small 0.716} \\ |
17 | 17 | \quad\textcolor{gray}{\small In-dist (Pro)} & \textcolor{gray}{\small 0.681} & \textcolor{gray}{\small 0.726} & \textcolor{gray}{\small 0.714} & \textcolor{gray}{\small 0.670} \\ |
18 | | -\quad Verified $\rightarrow$ Pro & 0.670\,{\scriptsize −0.011} & 0.693\,{\scriptsize −0.033} & 0.672\,{\scriptsize −0.042} & 0.616\,{\scriptsize −0.055} \\ |
19 | | -\quad Pro $\rightarrow$ Verified & 0.683\,{\scriptsize −0.092} & 0.742\,{\scriptsize −0.049} & 0.712\,{\scriptsize −0.057} & 0.650\,{\scriptsize −0.066} \\ |
| 18 | +\quad Verified $\rightarrow$ Pro & 0.670\,{\scriptsize $-0.011$} & 0.693\,{\scriptsize $-0.033$} & 0.672\,{\scriptsize $-0.042$} & 0.616\,{\scriptsize $-0.055$} \\ |
| 19 | +\quad Pro $\rightarrow$ Verified & 0.683\,{\scriptsize $-0.092$} & 0.742\,{\scriptsize $-0.049$} & 0.712\,{\scriptsize $-0.057$} & 0.650\,{\scriptsize $-0.066$} \\ |
20 | 20 | \midrule |
21 | 21 | \multicolumn{5}{l}{\textit{Reduces failures}} \\ |
22 | 22 | \quad\textcolor{gray}{\small In-dist (Verified)} & \textcolor{gray}{\small 0.743} & \textcolor{gray}{\small 0.769} & \textcolor{gray}{\small 0.743} & \textcolor{gray}{\small 0.674} \\ |
23 | 23 | \quad\textcolor{gray}{\small In-dist (Pro)} & \textcolor{gray}{\small 0.681} & \textcolor{gray}{\small 0.718} & \textcolor{gray}{\small 0.697} & \textcolor{gray}{\small 0.657} \\ |
24 | | -\quad Verified $\rightarrow$ Pro & 0.654\,{\scriptsize −0.026} & 0.690\,{\scriptsize −0.028} & 0.654\,{\scriptsize −0.043} & 0.573\,{\scriptsize −0.084} \\ |
25 | | -\quad Pro $\rightarrow$ Verified & 0.636\,{\scriptsize −0.107} & 0.684\,{\scriptsize −0.085} & 0.668\,{\scriptsize −0.075} & 0.610\,{\scriptsize −0.063} \\ |
| 24 | +\quad Verified $\rightarrow$ Pro & 0.654\,{\scriptsize $-0.026$} & 0.690\,{\scriptsize $-0.028$} & 0.654\,{\scriptsize $-0.043$} & 0.573\,{\scriptsize $-0.084$} \\ |
| 25 | +\quad Pro $\rightarrow$ Verified & 0.636\,{\scriptsize $-0.107$} & 0.684\,{\scriptsize $-0.085$} & 0.668\,{\scriptsize $-0.075$} & 0.610\,{\scriptsize $-0.063$} \\ |
26 | 26 | \midrule |
27 | 27 | \multicolumn{5}{l}{\textit{Has regressions}} \\ |
28 | 28 | \quad\textcolor{gray}{\small In-dist (Verified)} & \textcolor{gray}{\small 0.688} & \textcolor{gray}{\small 0.697} & \textcolor{gray}{\small 0.675} & \textcolor{gray}{\small 0.612} \\ |
29 | 29 | \quad\textcolor{gray}{\small In-dist (Pro)} & \textcolor{gray}{\small 0.645} & \textcolor{gray}{\small 0.705} & \textcolor{gray}{\small 0.739} & \textcolor{gray}{\small 0.691} \\ |
30 | | -\quad Verified $\rightarrow$ Pro & 0.560\,{\scriptsize −0.084} & 0.658\,{\scriptsize −0.047} & 0.634\,{\scriptsize −0.105} & 0.601\,{\scriptsize −0.090} \\ |
31 | | -\quad Pro $\rightarrow$ Verified & 0.583\,{\scriptsize −0.105} & 0.631\,{\scriptsize −0.066} & 0.604\,{\scriptsize −0.071} & 0.530\,{\scriptsize −0.081} \\ |
| 30 | +\quad Verified $\rightarrow$ Pro & 0.560\,{\scriptsize $-0.084$} & 0.658\,{\scriptsize $-0.047$} & 0.634\,{\scriptsize $-0.105$} & 0.601\,{\scriptsize $-0.090$} \\ |
| 31 | +\quad Pro $\rightarrow$ Verified & 0.583\,{\scriptsize $-0.105$} & 0.631\,{\scriptsize $-0.066$} & 0.604\,{\scriptsize $-0.071$} & 0.530\,{\scriptsize $-0.081$} \\ |
32 | 32 | \bottomrule |
33 | 33 | \end{tabular} |
34 | 34 | \end{table*} |
0 commit comments