Skip to content

Commit 24dd89b

Browse files
mberrmberr
committed
Optionally leave out std
1 parent d3a143f commit 24dd89b

5 files changed

Lines changed: 133 additions & 20 deletions

File tree

‎reproducibility/generate_summary_table.py‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ def load_published_results_in_tall_format(
6767
return pd.DataFrame(data=data, columns=['model', 'columns', 'mean', 'std'])
6868

6969

70-
def generate_results_table():
70+
def generate_results_table(with_std: bool):
7171
all_tables = []
7272
for dataset, dataset_df in read_experiment_collation().groupby('dataset'):
7373
if len(dataset_df['model'].unique()) < 2:
@@ -78,7 +78,7 @@ def generate_results_table():
7878

7979
published_results_tall_df = load_published_results_in_tall_format(dataset=dataset)
8080
all_results_tall_df = pd.concat([tall_summary_df, published_results_tall_df], ignore_index=True)
81-
wide_summary_df = reorganize_summary_df(all_results_tall_df)
81+
wide_summary_df = reorganize_summary_df(all_results_tall_df, with_std=with_std)
8282
wide_summary_df = wide_summary_df.applymap(lambda v: ('$' + v + '$') if v != '' else '')
8383

8484
# Save as Latex table
@@ -90,7 +90,8 @@ def generate_results_table():
9090
bold_rows=True,
9191
)
9292
# table_latex = _process_tex(table_latex)
93-
with open(os.path.join(SUMMARIES, f'{dataset}_table.tex'), 'w') as file:
93+
suffix = '' if with_std else '_without_std'
94+
with open(os.path.join(SUMMARIES, f'{dataset}_table{suffix}.tex'), 'w') as file:
9495
print(table_latex, file=file)
9596

9697
dataset = pykeen.datasets.datasets[dataset].__name__
@@ -133,7 +134,7 @@ def get_width(column):
133134
return prelen, postlen
134135

135136

136-
def format_values(mean, std, mean_w, std_w):
137+
def format_values(mean, std, mean_w, std_w, with_std: bool) -> str:
137138
if mean is None:
138139
return ''
139140
# format mean
@@ -145,31 +146,32 @@ def format_values(mean, std, mean_w, std_w):
145146
pre = r'\phantom{' + (pre_len - len(pre)) * '0' + '}' + pre
146147
if len(post) < post_len:
147148
post = post + r'\phantom{' + (post_len - len(post)) * '0' + '}'
148-
mean_str = pre + '.' + post
149-
if std is None:
150-
pre_len, post_len = std_w
151-
std_str = r'\phantom{ \pm ' + '0' * pre_len + '.' + '0' * post_len + '}'
152-
else:
153-
pre, post = std.split('.')
154-
pre_len, post_len = std_w
155-
if len(pre) < pre_len:
156-
pre = r'\phantom{' + (pre_len - len(pre)) * '0' + '}' + pre
157-
if len(post) < post_len:
158-
post = post + r'\phantom{' + (post_len - len(post)) * '0' + '}'
159-
std_str = r' \pm ' + pre + '.' + post
160-
out = mean_str + std_str
149+
out = pre + '.' + post
150+
if with_std:
151+
if std is None:
152+
pre_len, post_len = std_w
153+
std_str = r'\phantom{ \pm ' + '0' * pre_len + '.' + '0' * post_len + '}'
154+
else:
155+
pre, post = std.split('.')
156+
pre_len, post_len = std_w
157+
if len(pre) < pre_len:
158+
pre = r'\phantom{' + (pre_len - len(pre)) * '0' + '}' + pre
159+
if len(post) < post_len:
160+
post = post + r'\phantom{' + (post_len - len(post)) * '0' + '}'
161+
std_str = r' \pm ' + pre + '.' + post
162+
out += std_str
161163
out = out.replace(r'}\phantom{', '')
162164
return out
163165

164166

165-
def reorganize_summary_df(df: pd.DataFrame) -> pd.DataFrame:
167+
def reorganize_summary_df(df: pd.DataFrame, with_std: bool) -> pd.DataFrame:
166168
df['short_columns'] = df['columns'].str.replace('.((worst)|(avg)|(best)|(pub))', '', regex=True)
167169
_n = df.groupby(['short_columns']).aggregate({'mean': get_width, 'std': get_width})
168170
n = _n['std']
169171
_mean_n = _n['mean']
170172

171173
df['values'] = [
172-
format_values(mean=mean, std=std, mean_w=_mean_n[column], std_w=n[column])
174+
format_values(mean=mean, std=std, mean_w=_mean_n[column], std_w=n[column], with_std=with_std)
173175
for column, mean, std in df[['short_columns', 'mean', 'std']].values
174176
]
175177
inconsistent_rank_mask = df[~df['columns'].str.contains('.pub')].groupby(by=['model']).agg(
@@ -330,7 +332,8 @@ def generate_size_table():
330332

331333
def main():
332334
size_table = generate_size_table()
333-
all_tables = generate_results_table()
335+
for with_std in (False, True):
336+
all_tables = generate_results_table(with_std=with_std)
334337
write_pdfs(all_tables=all_tables, size_table=size_table)
335338

336339

‎reproducibility/summaries/fb15k237_table_without_std.tex‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
\begin{tabular}{llrrrrrrr}
2+
\toprule
3+
& & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\
4+
\textbf{model} & {} & & & & & & & \\
5+
\midrule
6+
\textbf{ConvE} & \textbf{pub} & $32.5\phantom{0}$ & $23.7\phantom{0}$ & $35.6\phantom{0}$ & & $50.1\phantom{0}$ & $\phantom{0}244.\phantom{00}$ & \\
7+
& \textbf{avg} & $26.93$ & $18.22$ & $29.51$ & $35.98$ & $44.95$ & $\phantom{0}255.46$ & $\phantom{0}3.73$ \\
8+
\textbf{ConvKB} & \textbf{pub} & $39.6\phantom{0}$ & & & & $51.7\phantom{0}$ & $\phantom{0}257.\phantom{00}$ & \\
9+
& \textbf{avg} & $\phantom{0}4.71$ & $\phantom{0}3.31$ & $\phantom{0}4.04$ & $\phantom{0}4.57$ & $\phantom{0}7.76$ & $4345.27$ & $61.36$ \\
10+
\textbf{RotatE} & \textbf{pub} & $33.8\phantom{0}$ & $24.1\phantom{0}$ & $37.5\phantom{0}$ & & $53.3\phantom{0}$ & $\phantom{0}177.\phantom{00}$ & \\
11+
& \textbf{avg} & $26.42$ & $17.57$ & $28.97$ & $35.29$ & $44.55$ & $\phantom{0}191.92$ & $\phantom{0}2.84$ \\
12+
\bottomrule
13+
\end{tabular}
14+

‎reproducibility/summaries/fb15k_table_without_std.tex‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
\begin{tabular}{llrrrrrrr}
2+
\toprule
3+
& & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\
4+
\textbf{model} & {} & & & & & & & \\
5+
\midrule
6+
\textbf{ComplEx} & \textbf{pub} & $69.2\phantom{0}$ & $59.9\phantom{0}$ & $75.9\phantom{0}$ & & $84.\phantom{00}$ & & \\
7+
& \textbf{avg} & $19.13$ & $10.06$ & $20.82$ & $27.59$ & $38.03$ & $\phantom{00}171.16$ & $\phantom{00}2.46$ \\
8+
\textbf{ConvE} & \textbf{pub} & $65.7\phantom{0}$ & $55.8\phantom{0}$ & $72.3\phantom{0}$ & & $83.1\phantom{0}$ & $\phantom{000}51.\phantom{00}$ & \\
9+
& \textbf{avg} & $59.56$ & $48.28$ & $66.99$ & $73.27$ & $79.76$ & $\phantom{000}50.76$ & $\phantom{00}0.73$ \\
10+
\textbf{DistMult} & \textbf{pub} & $35.\phantom{00}$ & & & & $57.7\phantom{0}$ & & \\
11+
& \textbf{avg} & $26.06$ & $16.45$ & $29.10$ & $35.54$ & $45.00$ & $\phantom{00}134.02$ & $\phantom{00}1.86$ \\
12+
\textbf{HolE} & \textbf{pub} & $52.4\phantom{0}$ & $40.2\phantom{0}$ & $61.3\phantom{0}$ & & $73.9\phantom{0}$ & & \\
13+
& \textbf{avg} & $34.15$ & $21.79$ & $39.69$ & $48.06$ & $58.84$ & $\phantom{00}193.03$ & $\phantom{00}2.71$ \\
14+
\textbf{KG2E} & \textbf{pub} & & & & & $71.5\phantom{0}$ & $\phantom{000}59.\phantom{00}$ & \\
15+
& \textbf{avg} & $\phantom{0}0.58$ & $\phantom{0}0.11$ & $\phantom{0}0.36$ & $\phantom{0}0.56$ & $\phantom{0}1.01$ & $\phantom{0}5779.07$ & $\phantom{0}78.40$ \\
16+
\textbf{RotatE} & \textbf{pub} & $79.7\phantom{0}$ & $74.6\phantom{0}$ & $83.\phantom{00}$ & & $88.4\phantom{0}$ & $\phantom{000}40.\phantom{00}$ & \\
17+
& \textbf{avg} & $55.00$ & $41.53$ & $64.14$ & $71.23$ & $78.67$ & $\phantom{000}42.28$ & $\phantom{00}0.63$ \\
18+
\textbf{SimplE} & \textbf{pub} & $72.7\phantom{0}$ & $66.\phantom{00}$ & $77.3\phantom{0}$ & & $83.8\phantom{0}$ & & \\
19+
& \textbf{avg} & $\phantom{0}0.04$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.06$ & $\phantom{0}7395.75$ & $100.02$ \\
20+
& \textbf{best} & $23.90$ & $11.58$ & $24.16$ & $34.73$ & $54.28$ & $\phantom{00}139.34$ & \\
21+
& \textbf{worst} & $\phantom{0}0.03$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.06$ & $14652.16$ & \\
22+
\textbf{TransD} & \textbf{pub} & & & & & $77.3\phantom{0}$ & $\phantom{000}91.\phantom{00}$ & \\
23+
& \textbf{avg} & $33.99$ & $21.22$ & $40.48$ & $48.57$ & $58.71$ & $\phantom{00}153.37$ & $\phantom{00}2.29$ \\
24+
\textbf{TransE} & \textbf{pub} & & & & & $47.1\phantom{0}$ & $\phantom{00}125.\phantom{00}$ & \\
25+
& \textbf{avg} & $26.01$ & $15.23$ & $29.85$ & $37.18$ & $47.34$ & $\phantom{00}127.92$ & $\phantom{00}1.78$ \\
26+
\textbf{TransH} & \textbf{pub} & & & & & $64.4\phantom{0}$ & $\phantom{000}87.\phantom{00}$ & \\
27+
& \textbf{avg} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.02$ & $\phantom{0}85.63$ \\
28+
& \textbf{best} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.00$ & \\
29+
& \textbf{worst} & $\phantom{0}2.54$ & $\phantom{0}1.69$ & $\phantom{0}2.95$ & $\phantom{0}3.29$ & $\phantom{0}3.74$ & $\phantom{0}6320.05$ & \\
30+
\textbf{TransR} & \textbf{pub} & & & & & $68.7\phantom{0}$ & $\phantom{000}77.\phantom{00}$ & \\
31+
& \textbf{avg} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.95$ & $\phantom{0}91.99$ \\
32+
& \textbf{best} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.94$ & \\
33+
& \textbf{worst} & $\phantom{0}0.65$ & $\phantom{0}0.37$ & $\phantom{0}0.63$ & $\phantom{0}0.78$ & $\phantom{0}1.03$ & $\phantom{0}6795.95$ & \\
34+
\textbf{TuckER} & \textbf{pub} & $79.5\phantom{0}$ & $74.1\phantom{0}$ & $83.3\phantom{0}$ & & $89.2\phantom{0}$ & & \\
35+
& \textbf{avg} & $\phantom{0}0.07$ & $\phantom{0}0.01$ & $\phantom{0}0.02$ & $\phantom{0}0.03$ & $\phantom{0}0.15$ & $\phantom{0}7327.77$ & $\phantom{0}99.11$ \\
36+
\bottomrule
37+
\end{tabular}
38+

‎reproducibility/summaries/wn18_table_without_std.tex‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
\begin{tabular}{llrrrrrrr}
2+
\toprule
3+
& & MRR (\%) & Hits@1 (\%) & Hits@3 (\%) & Hits@5 (\%) & Hits@10 (\%) & MR & AMR (\%) \\
4+
\textbf{model} & {} & & & & & & & \\
5+
\midrule
6+
\textbf{ComplEx} & \textbf{pub} & $94.1\phantom{0}$ & $93.6\phantom{0}$ & $94.5\phantom{0}$ & & $94.7\phantom{0}$ & & \\
7+
& \textbf{avg} & $19.49$ & $12.36$ & $20.66$ & $25.24$ & $32.92$ & $\phantom{00}452.67$ & $\phantom{00}2.21$ \\
8+
\textbf{ConvE} & \textbf{pub} & $94.3\phantom{0}$ & $93.5\phantom{0}$ & $94.6\phantom{0}$ & & $95.6\phantom{0}$ & $\phantom{00}374.\phantom{00}$ & \\
9+
& \textbf{avg} & $88.81$ & $85.14$ & $91.76$ & $93.29$ & $94.85$ & $\phantom{00}444.40$ & $\phantom{00}2.17$ \\
10+
\textbf{DistMult} & \textbf{pub} & $83.\phantom{00}$ & & & & $94.2\phantom{0}$ & & \\
11+
& \textbf{avg} & $77.44$ & $67.45$ & $85.94$ & $89.52$ & $92.72$ & $\phantom{00}458.64$ & $\phantom{00}2.24$ \\
12+
\textbf{HolE} & \textbf{pub} & $93.8\phantom{0}$ & $93.\phantom{00}$ & $94.5\phantom{0}$ & & $94.9\phantom{0}$ & & \\
13+
& \textbf{avg} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.64$ & $\phantom{00}3.97$ \\
14+
& \textbf{best} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.63$ & \\
15+
& \textbf{worst} & $70.44$ & $59.29$ & $79.29$ & $84.12$ & $88.61$ & $\phantom{00}812.64$ & \\
16+
\textbf{KG2E} & \textbf{pub} & & & & & $92.8\phantom{0}$ & $\phantom{00}331.\phantom{00}$ & \\
17+
& \textbf{avg} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.89$ & $\phantom{0}13.25$ \\
18+
& \textbf{best} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.88$ & \\
19+
& \textbf{worst} & $\phantom{0}3.61$ & $\phantom{0}1.35$ & $\phantom{0}3.21$ & $\phantom{0}4.57$ & $\phantom{0}7.02$ & $\phantom{0}2708.89$ & \\
20+
\textbf{RotatE} & \textbf{pub} & $94.9\phantom{0}$ & $94.4\phantom{0}$ & $95.2\phantom{0}$ & & $95.9\phantom{0}$ & $\phantom{00}309.\phantom{00}$ & \\
21+
& \textbf{avg} & $87.29$ & $82.17$ & $91.53$ & $93.44$ & $95.28$ & $\phantom{00}123.68$ & $\phantom{00}0.61$ \\
22+
\textbf{SimplE} & \textbf{pub} & $94.2\phantom{0}$ & $93.9\phantom{0}$ & $94.4\phantom{0}$ & & $94.7\phantom{0}$ & & \\
23+
& \textbf{avg} & $\phantom{0}0.04$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.07$ & $20376.43$ & $\phantom{0}99.57$ \\
24+
& \textbf{best} & $38.48$ & $33.93$ & $39.59$ & $42.76$ & $47.01$ & $\phantom{00}384.53$ & \\
25+
& \textbf{worst} & $\phantom{0}0.03$ & $\phantom{0}0.01$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $\phantom{0}0.07$ & $40368.33$ & \\
26+
\textbf{TransD} & \textbf{pub} & & & & & $92.2\phantom{0}$ & $\phantom{00}212.\phantom{00}$ & \\
27+
& \textbf{avg} & $36.22$ & $\phantom{0}3.94$ & $65.63$ & $79.64$ & $87.27$ & $\phantom{00}444.39$ & $\phantom{00}2.17$ \\
28+
\textbf{TransE} & \textbf{pub} & & & & & $89.2\phantom{0}$ & $\phantom{00}251.\phantom{00}$ & \\
29+
& \textbf{avg} & $39.19$ & $\phantom{0}9.99$ & $64.74$ & $75.44$ & $84.25$ & $\phantom{00}468.24$ & $\phantom{00}2.29$ \\
30+
\textbf{TransH} & \textbf{pub} & & & & & $82.3\phantom{0}$ & $\phantom{00}388.\phantom{00}$ & \\
31+
& \textbf{avg} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.04$ & $\phantom{0}96.16$ \\
32+
& \textbf{best} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.02$ & \\
33+
& \textbf{worst} & $\phantom{0}0.18$ & $\phantom{0}0.04$ & $\phantom{0}0.19$ & $\phantom{0}0.29$ & $\phantom{0}0.39$ & $19678.06$ & \\
34+
\textbf{TransR} & \textbf{pub} & & & & & $92.\phantom{00}$ & $\phantom{00}225.\phantom{00}$ & \\
35+
& \textbf{avg} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.49$ & $\phantom{0}96.20$ \\
36+
& \textbf{best} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.49$ & \\
37+
& \textbf{worst} & $\phantom{0}0.06$ & $\phantom{0}0.00$ & $\phantom{0}0.04$ & $\phantom{0}0.05$ & $\phantom{0}0.11$ & $19686.50$ & \\
38+
\textbf{TuckER} & \textbf{pub} & $95.3\phantom{0}$ & $94.9\phantom{0}$ & $95.5\phantom{0}$ & & $95.8\phantom{0}$ & & \\
39+
& \textbf{avg} & $\phantom{0}0.03$ & $\phantom{0}0.00$ & $\phantom{0}0.02$ & $\phantom{0}0.03$ & $\phantom{0}0.04$ & $20622.46$ & $100.78$ \\
40+
\bottomrule
41+
\end{tabular}
42+

0 commit comments

Comments
 (0)