Skip to content

Commit 441ccc8

Browse files
mberrmberr
committed
Add midrules
1 parent 24dd89b commit 441ccc8

9 files changed

Lines changed: 58 additions & 53 deletions

‎reproducibility/generate_summary_table.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ def load_published_results_in_tall_format(
6767
return pd.DataFrame(data=data, columns=['model', 'columns', 'mean', 'std'])
6868

6969

70-
def generate_results_table(with_std: bool):
70+
def generate_results_table(with_std: bool, midrule_between_models: bool = True):
7171
all_tables = []
7272
for dataset, dataset_df in read_experiment_collation().groupby('dataset'):
7373
if len(dataset_df['model'].unique()) < 2:
@@ -91,6 +91,11 @@ def generate_results_table(with_std: bool):
9191
)
9292
# table_latex = _process_tex(table_latex)
9393
suffix = '' if with_std else '_without_std'
94+
# add midrule
95+
if midrule_between_models:
96+
table_latex_head, table_latex_body = table_latex.split('\\midrule', maxsplit=1)
97+
table_latex_body = table_latex_body.replace('\n\\textbf', '\\midrule\n\\textbf')
98+
table_latex = table_latex_head + table_latex_body
9499
with open(os.path.join(SUMMARIES, f'{dataset}_table{suffix}.tex'), 'w') as file:
95100
print(table_latex, file=file)
96101

‎reproducibility/summaries/fb15k237_table.tex‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@
44
\textbf{model} & {} & & & & & & & \\
55
\midrule
66
\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0 \pm 0.00}$ & $23.7\phantom{0 \pm 0.00}$ & $35.6\phantom{0 \pm 0.00}$ & & $50.1\phantom{0 \pm 0.00}$ & $\phantom{0}244.\phantom{00 \pm 00.00}$ & \\
7-
& \textbf{avg} & $26.93 \pm 0.11$ & $18.22 \pm 0.11$ & $29.51 \pm 0.24$ & $35.98 \pm 0.16$ & $44.95 \pm 0.17$ & $\phantom{0}255.46 \pm \phantom{0}6.16$ & $\phantom{0}3.73 \pm 0.13$ \\
7+
& \textbf{avg} & $26.93 \pm 0.11$ & $18.22 \pm 0.11$ & $29.51 \pm 0.24$ & $35.98 \pm 0.16$ & $44.95 \pm 0.17$ & $\phantom{0}255.46 \pm \phantom{0}6.16$ & $\phantom{0}3.73 \pm 0.13$ \\\midrule
88
\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0 \pm 0.00}$ & & & & $51.7\phantom{0 \pm 0.00}$ & $\phantom{0}257.\phantom{00 \pm 00.00}$ & \\
9-
& \textbf{avg} & $\phantom{0}4.71 \pm 0.23$ & $\phantom{0}3.31 \pm 0.23$ & $\phantom{0}4.04 \pm 0.19$ & $\phantom{0}4.57 \pm 0.22$ & $\phantom{0}7.76 \pm 0.88$ & $4345.27 \pm 46.99$ & $61.36 \pm 0.65$ \\
9+
& \textbf{avg} & $\phantom{0}4.71 \pm 0.23$ & $\phantom{0}3.31 \pm 0.23$ & $\phantom{0}4.04 \pm 0.19$ & $\phantom{0}4.57 \pm 0.22$ & $\phantom{0}7.76 \pm 0.88$ & $4345.27 \pm 46.99$ & $61.36 \pm 0.65$ \\\midrule
1010
\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0 \pm 0.00}$ & $24.1\phantom{0 \pm 0.00}$ & $37.5\phantom{0 \pm 0.00}$ & & $53.3\phantom{0 \pm 0.00}$ & $\phantom{0}177.\phantom{00 \pm 00.00}$ & \\
1111
& \textbf{avg} & $26.42 \pm 0.04$ & $17.57 \pm 0.06$ & $28.97 \pm 0.05$ & $35.29 \pm 0.09$ & $44.55 \pm 0.06$ & $\phantom{0}191.92 \pm \phantom{0}0.31$ & $\phantom{0}2.84 \pm 0.00$ \\
1212
\bottomrule

‎reproducibility/summaries/fb15k237_table_without_std.tex‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@
44
\textbf{model} & {} & & & & & & & \\
55
\midrule
66
\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0}$ & $23.7\phantom{0}$ & $35.6\phantom{0}$ & & $50.1\phantom{0}$ & $\phantom{0}244.\phantom{00}$ & \\
7-
& \textbf{avg} & $26.93$ & $18.22$ & $29.51$ & $35.98$ & $44.95$ & $\phantom{0}255.46$ & $\phantom{0}3.73$ \\
7+
& \textbf{avg} & $26.93$ & $18.22$ & $29.51$ & $35.98$ & $44.95$ & $\phantom{0}255.46$ & $\phantom{0}3.73$ \\\midrule
88
\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0}$ & & & & $51.7\phantom{0}$ & $\phantom{0}257.\phantom{00}$ & \\
9-
& \textbf{avg} & $\phantom{0}4.71$ & $\phantom{0}3.31$ & $\phantom{0}4.04$ & $\phantom{0}4.57$ & $\phantom{0}7.76$ & $4345.27$ & $61.36$ \\
9+
& \textbf{avg} & $\phantom{0}4.71$ & $\phantom{0}3.31$ & $\phantom{0}4.04$ & $\phantom{0}4.57$ & $\phantom{0}7.76$ & $4345.27$ & $61.36$ \\\midrule
1010
\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0}$ & $24.1\phantom{0}$ & $37.5\phantom{0}$ & & $53.3\phantom{0}$ & $\phantom{0}177.\phantom{00}$ & \\
1111
& \textbf{avg} & $26.42$ & $17.57$ & $28.97$ & $35.29$ & $44.55$ & $\phantom{0}191.92$ & $\phantom{0}2.84$ \\
1212
\bottomrule

‎reproducibility/summaries/fb15k_table.tex‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,33 +4,33 @@
44
\textbf{model} & {} & & & & & & & \\
55
\midrule
66
\textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0 \pm 0.00}$ & $59.9\phantom{0 \pm 0.00}$ & $75.9\phantom{0 \pm 00.00}$ & & $84.\phantom{00 \pm 00.00}$ & & \\
7-
& \textbf{avg} & $19.13 \pm 0.45$ & $10.06 \pm 0.42$ & $20.82 \pm \phantom{0}0.67$ & $27.59 \pm \phantom{0}0.94$ & $38.03 \pm \phantom{0}1.02$ & $\phantom{00}171.16 \pm 16.05$ & $\phantom{00}2.46 \pm 0.28$ \\
7+
& \textbf{avg} & $19.13 \pm 0.45$ & $10.06 \pm 0.42$ & $20.82 \pm \phantom{0}0.67$ & $27.59 \pm \phantom{0}0.94$ & $38.03 \pm \phantom{0}1.02$ & $\phantom{00}171.16 \pm 16.05$ & $\phantom{00}2.46 \pm 0.28$ \\\midrule
88
\textbf{ConvE} & \textbf{pub} & $65.7\phantom{0 \pm 0.00}$ & $55.8\phantom{0 \pm 0.00}$ & $72.3\phantom{0 \pm 00.00}$ & & $83.1\phantom{0 \pm 00.00}$ & $\phantom{000}51.\phantom{00 \pm 00.00}$ & \\
9-
& \textbf{avg} & $59.56 \pm 0.06$ & $48.28 \pm 0.12$ & $66.99 \pm \phantom{0}0.04$ & $73.27 \pm \phantom{0}0.03$ & $79.76 \pm \phantom{0}0.07$ & $\phantom{000}50.76 \pm \phantom{0}0.40$ & $\phantom{00}0.73 \pm 0.01$ \\
9+
& \textbf{avg} & $59.56 \pm 0.06$ & $48.28 \pm 0.12$ & $66.99 \pm \phantom{0}0.04$ & $73.27 \pm \phantom{0}0.03$ & $79.76 \pm \phantom{0}0.07$ & $\phantom{000}50.76 \pm \phantom{0}0.40$ & $\phantom{00}0.73 \pm 0.01$ \\\midrule
1010
\textbf{DistMult} & \textbf{pub} & $35.\phantom{00 \pm 0.00}$ & & & & $57.7\phantom{0 \pm 00.00}$ & & \\
11-
& \textbf{avg} & $26.06 \pm 0.17$ & $16.45 \pm 0.16$ & $29.10 \pm \phantom{0}0.17$ & $35.54 \pm \phantom{0}0.21$ & $45.00 \pm \phantom{0}0.25$ & $\phantom{00}134.02 \pm \phantom{0}1.98$ & $\phantom{00}1.86 \pm 0.03$ \\
11+
& \textbf{avg} & $26.06 \pm 0.17$ & $16.45 \pm 0.16$ & $29.10 \pm \phantom{0}0.17$ & $35.54 \pm \phantom{0}0.21$ & $45.00 \pm \phantom{0}0.25$ & $\phantom{00}134.02 \pm \phantom{0}1.98$ & $\phantom{00}1.86 \pm 0.03$ \\\midrule
1212
\textbf{HolE} & \textbf{pub} & $52.4\phantom{0 \pm 0.00}$ & $40.2\phantom{0 \pm 0.00}$ & $61.3\phantom{0 \pm 00.00}$ & & $73.9\phantom{0 \pm 00.00}$ & & \\
13-
& \textbf{avg} & $34.15 \pm 0.22$ & $21.79 \pm 0.19$ & $39.69 \pm \phantom{0}0.24$ & $48.06 \pm \phantom{0}0.30$ & $58.84 \pm \phantom{0}0.28$ & $\phantom{00}193.03 \pm \phantom{0}7.61$ & $\phantom{00}2.71 \pm 0.12$ \\
13+
& \textbf{avg} & $34.15 \pm 0.22$ & $21.79 \pm 0.19$ & $39.69 \pm \phantom{0}0.24$ & $48.06 \pm \phantom{0}0.30$ & $58.84 \pm \phantom{0}0.28$ & $\phantom{00}193.03 \pm \phantom{0}7.61$ & $\phantom{00}2.71 \pm 0.12$ \\\midrule
1414
\textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0 \pm 00.00}$ & $\phantom{000}59.\phantom{00 \pm 00.00}$ & \\
15-
& \textbf{avg} & $\phantom{0}0.58 \pm 0.07$ & $\phantom{0}0.11 \pm 0.04$ & $\phantom{0}0.36 \pm \phantom{0}0.08$ & $\phantom{0}0.56 \pm \phantom{0}0.10$ & $\phantom{0}1.01 \pm \phantom{0}0.14$ & $\phantom{0}5779.07 \pm 51.02$ & $\phantom{0}78.40 \pm 0.68$ \\
15+
& \textbf{avg} & $\phantom{0}0.58 \pm 0.07$ & $\phantom{0}0.11 \pm 0.04$ & $\phantom{0}0.36 \pm \phantom{0}0.08$ & $\phantom{0}0.56 \pm \phantom{0}0.10$ & $\phantom{0}1.01 \pm \phantom{0}0.14$ & $\phantom{0}5779.07 \pm 51.02$ & $\phantom{0}78.40 \pm 0.68$ \\\midrule
1616
\textbf{RotatE} & \textbf{pub} & $79.7\phantom{0 \pm 0.00}$ & $74.6\phantom{0 \pm 0.00}$ & $83.\phantom{00 \pm 00.00}$ & & $88.4\phantom{0 \pm 00.00}$ & $\phantom{000}40.\phantom{00 \pm 00.00}$ & \\
17-
& \textbf{avg} & $55.00 \pm 0.06$ & $41.53 \pm 0.06$ & $64.14 \pm \phantom{0}0.07$ & $71.23 \pm \phantom{0}0.05$ & $78.67 \pm \phantom{0}0.08$ & $\phantom{000}42.28 \pm \phantom{0}0.13$ & $\phantom{00}0.63 \pm 0.00$ \\
17+
& \textbf{avg} & $55.00 \pm 0.06$ & $41.53 \pm 0.06$ & $64.14 \pm \phantom{0}0.07$ & $71.23 \pm \phantom{0}0.05$ & $78.67 \pm \phantom{0}0.08$ & $\phantom{000}42.28 \pm \phantom{0}0.13$ & $\phantom{00}0.63 \pm 0.00$ \\\midrule
1818
\textbf{SimplE} & \textbf{pub} & $72.7\phantom{0 \pm 0.00}$ & $66.\phantom{00 \pm 0.00}$ & $77.3\phantom{0 \pm 00.00}$ & & $83.8\phantom{0 \pm 00.00}$ & & \\
1919
& \textbf{avg} & $\phantom{0}0.04 \pm 0.00$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.00$ & $\phantom{0}0.04 \pm \phantom{0}0.00$ & $\phantom{0}0.06 \pm \phantom{0}0.01$ & $\phantom{0}7395.75 \pm \phantom{0}2.02$ & $100.02 \pm 0.03$ \\
2020
& \textbf{best} & $23.90 \pm 8.79$ & $11.58 \pm 6.42$ & $24.16 \pm 10.95$ & $34.73 \pm 13.40$ & $54.28 \pm 15.80$ & $\phantom{00}139.34 \pm 49.45$ & \\
21-
& \textbf{worst} & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.00$ & $\phantom{0}0.04 \pm \phantom{0}0.00$ & $\phantom{0}0.06 \pm \phantom{0}0.01$ & $14652.16 \pm 45.71$ & \\
21+
& \textbf{worst} & $\phantom{0}0.03 \pm 0.00$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.00$ & $\phantom{0}0.04 \pm \phantom{0}0.00$ & $\phantom{0}0.06 \pm \phantom{0}0.01$ & $14652.16 \pm 45.71$ & \\\midrule
2222
\textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0 \pm 00.00}$ & $\phantom{000}91.\phantom{00 \pm 00.00}$ & \\
23-
& \textbf{avg} & $33.99 \pm 0.03$ & $21.22 \pm 0.03$ & $40.48 \pm \phantom{0}0.10$ & $48.57 \pm \phantom{0}0.09$ & $58.71 \pm \phantom{0}0.14$ & $\phantom{00}153.37 \pm \phantom{0}5.35$ & $\phantom{00}2.29 \pm 0.09$ \\
23+
& \textbf{avg} & $33.99 \pm 0.03$ & $21.22 \pm 0.03$ & $40.48 \pm \phantom{0}0.10$ & $48.57 \pm \phantom{0}0.09$ & $58.71 \pm \phantom{0}0.14$ & $\phantom{00}153.37 \pm \phantom{0}5.35$ & $\phantom{00}2.29 \pm 0.09$ \\\midrule
2424
\textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0 \pm 00.00}$ & $\phantom{00}125.\phantom{00 \pm 00.00}$ & \\
25-
& \textbf{avg} & $26.01 \pm 0.17$ & $15.23 \pm 0.16$ & $29.85 \pm \phantom{0}0.24$ & $37.18 \pm \phantom{0}0.24$ & $47.34 \pm \phantom{0}0.18$ & $\phantom{00}127.92 \pm \phantom{0}0.86$ & $\phantom{00}1.78 \pm 0.01$ \\
25+
& \textbf{avg} & $26.01 \pm 0.17$ & $15.23 \pm 0.16$ & $29.85 \pm \phantom{0}0.24$ & $37.18 \pm \phantom{0}0.24$ & $47.34 \pm \phantom{0}0.18$ & $\phantom{00}127.92 \pm \phantom{0}0.86$ & $\phantom{00}1.78 \pm 0.01$ \\\midrule
2626
\textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0 \pm 00.00}$ & $\phantom{000}87.\phantom{00 \pm 00.00}$ & \\
2727
& \textbf{avg} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.02 \pm 30.37$ & $\phantom{0}85.63 \pm 0.40$ \\
2828
& \textbf{best} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.00 \pm 30.37$ & \\
29-
& \textbf{worst} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.05 \pm 30.37$ & \\
29+
& \textbf{worst} & $\phantom{0}2.54 \pm 0.20$ & $\phantom{0}1.69 \pm 0.25$ & $\phantom{0}2.95 \pm \phantom{0}0.20$ & $\phantom{0}3.29 \pm \phantom{0}0.22$ & $\phantom{0}3.74 \pm \phantom{0}0.18$ & $\phantom{0}6320.05 \pm 30.37$ & \\\midrule
3030
\textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0 \pm 00.00}$ & $\phantom{000}77.\phantom{00 \pm 00.00}$ & \\
3131
& \textbf{avg} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.95 \pm 16.65$ & $\phantom{0}91.99 \pm 0.22$ \\
3232
& \textbf{best} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.94 \pm 16.65$ & \\
33-
& \textbf{worst} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.95 \pm 16.65$ & \\
33+
& \textbf{worst} & $\phantom{0}0.65 \pm 0.02$ & $\phantom{0}0.37 \pm 0.00$ & $\phantom{0}0.63 \pm \phantom{0}0.04$ & $\phantom{0}0.78 \pm \phantom{0}0.06$ & $\phantom{0}1.03 \pm \phantom{0}0.07$ & $\phantom{0}6795.95 \pm 16.65$ & \\\midrule
3434
\textbf{TuckER} & \textbf{pub} & $79.5\phantom{0 \pm 0.00}$ & $74.1\phantom{0 \pm 0.00}$ & $83.3\phantom{0 \pm 00.00}$ & & $89.2\phantom{0 \pm 00.00}$ & & \\
3535
& \textbf{avg} & $\phantom{0}0.07 \pm 0.02$ & $\phantom{0}0.01 \pm 0.00$ & $\phantom{0}0.02 \pm \phantom{0}0.00$ & $\phantom{0}0.03 \pm \phantom{0}0.01$ & $\phantom{0}0.15 \pm \phantom{0}0.17$ & $\phantom{0}7327.77 \pm 29.22$ & $\phantom{0}99.11 \pm 0.39$ \\
3636
\bottomrule

0 commit comments

Comments
 (0)