WebDetective/wikieval_analysis.py at main · Alibaba-NLP/WebDetective · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
import json
import pandas as pd

# If you need to print the results to the console, set to True; otherwise False
verbose = False

if verbose:
    from prettytable import PrettyTable
    table = PrettyTable()
    table.field_names = ["Dataset", "Model",
        "Avg. Search Score@1", "Best Search Score@1",
        "Avg. Generation Score@1", "Best Generation Score@1",
        "Avg. Good Refusal F1@1", "Best Good Refusal F1@1",
        "Avg. Knowledge Utilisation F1@1", "Best Knowledge Utilisation F1@1",
        "Avg. Pass@1", "Best Pass@1", "Pass@3",
        "Avg. Knowledge Sufficiency@1", "Best Knowledge Sufficiency@1",
        "Avg. RAG Knowledge Sufficiency@1", "Best RAG Knowledge Sufficiency@1",
        "Avg. Knowledge Forget@1", "Best Knowledge Forget@1",
        "Avg. Knowledge Leadastray@1", "Best Knowledge Leadastray@1",
        "Avg. Is Refusal", "Best Is Refusal",
        "Avg. Visit Tool Count", "Avg. Tool Use Count", "Avg. Optimal Hop Count"]

dataset = "mask_wiki_200.jsonl"
summary_path = f"path/to/WebDetective/eval_out/{dataset}_summary.jsonl"
excel_path  = f"path/to/WebDetective/eval_out/{dataset}_summary.xlsx"

# Used to store all the rows of data
rows = []

for line in open(summary_path, 'r', encoding='utf-8'):
    raw = json.loads(line)
    # Extract model name from files - try round1 first, fallback to any round
    model_name = ""
    if "files" in raw:
        for round_name, file_path in raw["files"].items():
            if file_path:
                model_name = file_path.split("/")[-3].replace("_sglang", "")
                break

    # Handle different evaluation modes
    evaluation_mode = raw.get("evaluation_mode", "simple")

    if evaluation_mode == "holistic":
        # Holistic evaluation metrics
        scores = {
            "Avg. Search Score@1":   raw["holistic_metrics"].get("avg_search_score", 0),
            "Best Search Score@1":  raw["holistic_metrics"].get("best_search_score", 0),
            "Avg. Generation Score@1":  raw["holistic_metrics"].get("avg_gen_score", 0),
            "Best Generation Score@1":  raw["holistic_metrics"].get("best_gen_score", 0),
            "Avg. Good Refusal F1@1": raw["holistic_metrics"].get("avg_good_refusal_f1", 0),
            "Best Good Refusal F1@1": raw["holistic_metrics"].get("best_good_refusal_f1", 0),
            "Avg. Knowledge Utilisation F1@1": raw["holistic_metrics"].get("avg_knowledge_utilisation_f1", 0),
            "Best Knowledge Utilisation F1@1": raw["holistic_metrics"].get("best_knowledge_utilisation_f1", 0),
            "Avg. Pass@1":       raw["holistic_metrics"].get("avg_pass_at_1", 0),
            "Best Pass@1":       raw["holistic_metrics"].get("best_pass_at_1", 0),
            "Pass@3":            raw["holistic_metrics"].get("pass_at_k", 0),
            "Avg. Knowledge Sufficiency@1": raw["holistic_metrics"].get("avg_knowledge_sufficient", 0),
            "Avg. RAG Knowledge Sufficiency@1": raw["holistic_metrics"].get("avg_rag_knowledge_sufficient", 0),
            "Avg. Knowledge Forget@1": raw["holistic_metrics"].get("avg_knowledge_forget_to_use", 0),
            "Best Knowledge Forget@1": raw["holistic_metrics"].get("best_knowledge_forget_to_use", 0),
            "Avg. Knowledge Leadastray@1": raw["holistic_metrics"].get("avg_knowledge_leadastray", 0),
            "Best Knowledge Leadastray@1": raw["holistic_metrics"].get("best_knowledge_leadastray", 0),
            "Avg. Is Refusal": raw["holistic_metrics"].get("avg_is_refusal", 0),
            "Best Is Refusal": raw["holistic_metrics"].get("best_is_refusal", 0),
        }
    else:
        # Simple evaluation - use overall metrics
        scores = {
            "Avg. Search Score@1":   0,  # Not available in simple mode
            "Best Search Score@1":  0,  # Not available in simple mode
            "Avg. Generation Score@1":  0,  # Not available in simple mode
            "Best Generation Score@1":  0,  # Not available in simple mode
            "Avg. Good Refusal F1@1": 0,  # Not available in simple mode
            "Best Good Refusal F1@1": 0,  # Not available in simple mode
            "Avg. Knowledge Utilisation F1@1": 0,  # Not available in simple mode
            "Best Knowledge Utilisation F1@1": 0,  # Not available in simple mode
            "Avg. Knowledge Forget@1": 0,  # Not available in simple mode
            "Best Knowledge Forget@1": 0,  # Not available in simple mode
            "Avg. Knowledge Leadastray@1": 0,  # Not available in simple mode
            "Best Knowledge Leadastray@1": 0,  # Not available in simple mode
            "Avg. Pass@1":       raw["overall"].get("avg_pass_at_1", 0),
            "Best Pass@1":       raw["overall"].get("best_pass_at_1", 0),
            "Pass@3":            raw["overall"].get("pass_at_k", 0),
            "Avg. Knowledge Sufficiency@1": 0,  # Not available in simple mode
            "Avg. RAG Knowledge Sufficiency@1": 0,  # Not available in simple mode
            "Avg. Is Refusal": 0,  # Not available in simple mode
        }

    # Statistics are common across both modes
    stats = {
        "Avg. answer":        raw["statistics"].get("avg_ans_length", 0),
        "Avg. think":         raw["statistics"].get("avg_think_length", 0),
        "Avg. action":        raw["statistics"].get("avg_action", 0),
        "Avg. visit action":  raw["statistics"].get("avg_visit_action", 0),
        "Avg. search action": raw["statistics"].get("avg_search_action", 0),
        "Avg. other action":  raw["statistics"].get("avg_other_action", 0),
    }

    # Additional holistic metrics if available
    additional_metrics = {}
    if evaluation_mode == "holistic" and "holistic_metrics" in raw:
        additional_metrics = {
            "Avg. Visit Tool Count": raw["holistic_metrics"].get("avg_visit_tool_count", 0),
            "Avg. Tool Use Count": raw["holistic_metrics"].get("avg_tool_use_count", 0),
            "Avg. Optimal Hop Count": raw["holistic_metrics"].get("avg_optimal_hop_count", 0),
        }
    else:
        additional_metrics = {
            "Avg. Visit Tool Count": 0,
            "Avg. Tool Use Count": 0,
            "Avg. Optimal Hop Count": 0,
        }

    row = {
        "Dataset": dataset,
        "Model":   model_name,
        **scores,
        **additional_metrics,
        **stats,
    }
    rows.append(row)

# Construct DataFrame
df = pd.DataFrame(rows)

# Save to Excel
df.to_excel(excel_path, index=False)
print(f"Saved summary to Excel: {excel_path}")

# If you need to print the sorted table to the console
if verbose:
    # Sort by Best Pass@1 (the 6th column, index 3) in descending order
    df_sorted = df.sort_values(by="Best Pass@1", ascending=False)
    for _, r in df_sorted.iterrows():
        table.add_row([
            r["Dataset"], r["Model"],
            r["Avg. Search Score@1"], r["Best Search Score@1"],
            r["Avg. Generation Score@1"], r["Best Generation Score@1"],
            r["Avg. Good Refusal F1@1"], r["Best Good Refusal F1@1"],
            r["Avg. Knowledge Utilisation F1@1"], r["Best Knowledge Utilisation F1@1"],
            r["Avg. Knowledge Forget@1"], r["Best Knowledge Forget@1"],
            r["Avg. Knowledge Leadastray@1"], r["Best Knowledge Leadastray@1"],
            r["Avg. Pass@1"], r["Best Pass@1"], r["Pass@3"],
            r["Avg. Knowledge Sufficiency@1"], r["Avg. RAG Knowledge Sufficiency@1"],
            r["Avg. answer"], r["Avg. think"], r["Avg. Visit Tool Count"], r["Avg. Optimal Hop Count"]
        ])
    print(table)