Skip to content

Commit 81d637d

Browse files
codewiththomascodewiththomas
committed
Restructed folder; Added files for evaluation;
1 parent 6d9c7f3 commit 81d637d

42 files changed

Lines changed: 165 additions & 2 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 4 additions & 2 deletions

‎eval/evaluation.ipynb‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"# Evaluierung der Ergebnisse"
8+
]
9+
},
10+
{
11+
"cell_type": "code",
12+
"execution_count": null,
13+
"metadata": {},
14+
"outputs": [
15+
{
16+
"name": "stdout",
17+
"output_type": "stream",
18+
"text": [
19+
" METRIK RAG TF-IDF GPT\n",
20+
"0 PRECISION@5 0.825 0.630 NaN\n",
21+
"1 RECALL@5 0.700 0.465 NaN\n",
22+
"2 ANTWORTRELEVANZ 4.125 3.125 3.350\n",
23+
"3 FAKTENTREUE 4.300 3.050 2.300\n",
24+
"4 VOLLSTÄNDIGKEIT 3.750 2.575 2.125\n"
25+
]
26+
}
27+
],
28+
"source": [
29+
"import pandas as pd\n",
30+
"\n",
31+
"file_path = \"evaluation_anonym.csv\" #same dir as notbook\n",
32+
"df = pd.read_csv(file_path, sep=\",\")\n",
33+
"\n",
34+
"metrics_avg = {\n",
35+
" \"METRIK\": [\"PRECISION@5\", \"RECALL@5\", \"ANTWORTRELEVANZ\", \"FAKTENTREUE\", \"VOLLSTÄNDIGKEIT\"],\n",
36+
" \"RAG\": [\n",
37+
" df[\"RAG_Precision5\"].mean(),\n",
38+
" df[\"RAG_Recall5\"].mean(),\n",
39+
" df[\"RAG_Antwortrelevanz\"].mean(),\n",
40+
" df[\"RAG_Faktentreue\"].mean(),\n",
41+
" df[\"RAG_Vollständigkeit\"].mean()\n",
42+
" ],\n",
43+
" \"TF-IDF\": [\n",
44+
" df[\"Baseline_Precision5\"].mean(),\n",
45+
" df[\"Baseline_Recall5\"].mean(),\n",
46+
" df[\"Baseline_Antwortrelevanz\"].mean(),\n",
47+
" df[\"Baseline_Faktentreue\"].mean(),\n",
48+
" df[\"Baseline_Vollständigkeit\"].mean()\n",
49+
" ],\n",
50+
" \"GPT\": [\n",
51+
" None, # Kein Precision@5 für GPT in den Daten vorhanden\n",
52+
" None, # Kein Recall@5 für GPT in den Daten vorhanden\n",
53+
" df[\"GPT_Antwortrelevanz\"].mean(),\n",
54+
" df[\"GPT_Faktentreue\"].mean(),\n",
55+
" df[\"GPT_Vollständigkeit\"].mean()\n",
56+
" ]\n",
57+
"}\n",
58+
"\n",
59+
"metrics_df = pd.DataFrame(metrics_avg)\n",
60+
"\n",
61+
"print(metrics_df)"
62+
]
63+
}
64+
],
65+
"metadata": {
66+
"kernelspec": {
67+
"display_name": ".venv",
68+
"language": "python",
69+
"name": "python3"
70+
},
71+
"language_info": {
72+
"codemirror_mode": {
73+
"name": "ipython",
74+
"version": 3
75+
},
76+
"file_extension": ".py",
77+
"mimetype": "text/x-python",
78+
"name": "python",
79+
"nbconvert_exporter": "python",
80+
"pygments_lexer": "ipython3",
81+
"version": "3.12.7"
82+
}
83+
},
84+
"nbformat": 4,
85+
"nbformat_minor": 2
86+
}

‎eval/evaluation_anonym.csv‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
Frage_ID,Kategorie,Frage,Gold_Standard,RAG_Precision5,RAG_Recall5,RAG_Antwortrelevanz,RAG_Faktentreue,RAG_Vollständigkeit,Baseline_Precision5,Baseline_Recall5,Baseline_Antwortrelevanz,Baseline_Faktentreue,Baseline_Vollständigkeit,GPT_Antwortrelevanz,GPT_Faktentreue,GPT_Vollständigkeit
2+
F001,Faktenfrage,****,****,0.8,0.8,4,4,4,0.8,0.6,3,3,3,4,2,2
3+
F002,Faktenfrage,****,****,1.0,0.8,5,5,4,0.8,0.6,4,3,3,3,2,2
4+
F003,Faktenfrage,****,****,0.8,0.8,4,5,4,0.6,0.6,3,3,3,4,3,2
5+
F004,Faktenfrage,****,****,0.8,0.6,4,4,4,0.6,0.4,3,3,2,3,2,2
6+
F005,Faktenfrage,****,****,0.8,0.6,4,4,3,0.6,0.6,3,3,3,3,2,2
7+
F006,Faktenfrage,****,****,1.0,0.8,5,5,4,0.8,0.6,4,4,3,4,3,2
8+
F007,Faktenfrage,****,****,0.8,0.8,4,4,4,0.6,0.4,3,3,3,3,2,2
9+
F008,Faktenfrage,****,****,0.8,0.6,4,5,4,0.6,0.4,3,3,2,4,3,2
10+
F009,Faktenfrage,****,****,0.8,0.8,4,4,4,0.6,0.6,3,3,3,3,2,2
11+
F010,Faktenfrage,****,****,1.0,0.8,5,5,4,0.8,0.6,4,4,3,4,3,2
12+
F011,Faktenfrage,****,****,0.8,0.8,4,4,4,0.6,0.4,3,3,2,3,2,2
13+
F012,Faktenfrage,****,****,0.8,0.6,4,5,4,0.6,0.4,3,3,3,3,2,2
14+
F013,Faktenfrage,****,****,0.8,0.6,4,4,3,0.6,0.4,3,3,2,3,2,2
15+
F014,Faktenfrage,****,****,1.0,0.8,5,5,4,0.8,0.6,4,4,3,4,2,2
16+
F015,Faktenfrage,****,****,0.8,0.8,4,4,4,0.6,0.4,3,3,3,3,2,2
17+
F016,Faktenfrage,****,****,0.8,0.6,4,5,4,0.6,0.6,3,3,3,3,2,2
18+
F017,Faktenfrage,****,****,0.8,0.8,4,4,4,0.6,0.4,3,3,2,3,2,2
19+
F018,Faktenfrage,****,****,1.0,0.8,5,5,4,0.8,0.6,4,3,3,4,3,2
20+
F019,Faktenfrage,****,****,0.8,0.6,4,4,3,0.6,0.4,3,3,3,3,2,2
21+
F020,Faktenfrage,****,****,0.8,0.6,4,5,4,0.6,0.4,3,3,3,3,2,2
22+
F021,Faktenfrage,****,****,0.8,0.8,4,4,4,0.6,0.4,3,3,2,3,2,2
23+
F022,Faktenfrage,****,****,1.0,0.8,5,5,4,0.8,0.6,4,4,3,4,3,2
24+
F023,Faktenfrage,****,****,0.8,0.8,4,4,4,0.6,0.4,3,3,3,3,2,2
25+
F024,Faktenfrage,****,****,0.8,0.6,4,5,4,0.6,0.4,3,3,3,3,2,2
26+
F025,Faktenfrage,****,****,0.8,0.6,4,4,3,0.6,0.4,3,3,2,3,2,2
27+
F026,Faktenfrage,****,****,0.8,0.6,4,5,4,0.6,0.4,3,3,3,3,2,2
28+
F027,Faktenfrage,****,****,0.8,0.8,4,4,4,0.6,0.6,3,3,3,3,2,2
29+
P001,Prozessfrage,****,****,0.8,0.6,4,4,4,0.6,0.4,3,3,2,4,3,2
30+
P002,Prozessfrage,****,****,0.8,0.8,4,4,4,0.6,0.4,3,3,2,3,2,2
31+
P003,Prozessfrage,****,****,1.0,0.8,5,4,4,0.8,0.6,4,3,3,4,3,3
32+
P004,Prozessfrage,****,****,0.8,0.8,4,4,4,0.6,0.4,3,3,2,3,2,2
33+
P005,Prozessfrage,****,****,0.8,0.6,4,4,4,0.6,0.4,3,3,3,3,2,2
34+
P006,Prozessfrage,****,****,0.8,0.6,4,4,3,0.6,0.4,3,3,2,3,2,2
35+
P007,Prozessfrage,****,****,0.8,0.6,4,4,4,0.6,0.4,3,3,3,4,3,3
36+
P008,Prozessfrage,****,****,0.8,0.8,4,4,4,0.6,0.4,3,3,2,3,2,2
37+
L001,Problemlösungsfrage,****,****,0.8,0.6,4,4,3,0.6,0.4,3,3,2,4,3,3
38+
L002,Problemlösungsfrage,****,****,0.6,0.6,3,4,3,0.4,0.4,2,2,2,3,2,2
39+
L003,Problemlösungsfrage,****,****,0.8,0.6,4,4,3,0.6,0.4,3,3,2,4,3,3
40+
L004,Problemlösungsfrage,****,****,0.6,0.6,3,3,3,0.4,0.4,2,2,2,3,2,2
41+
L005,Problemlösungsfrage,****,****,0.8,0.6,4,4,3,0.6,0.4,3,3,2,4,3,3

‎eval/requirements.txt‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
ace_tools==0.0
2+
asttokens==3.0.0
3+
colorama==0.4.6
4+
comm==0.2.2
5+
debugpy==1.8.12
6+
decorator==5.2.1
7+
executing==2.2.0
8+
ipykernel==6.29.5
9+
ipython==9.0.0
10+
ipython_pygments_lexers==1.1.1
11+
jedi==0.19.2
12+
jupyter_client==8.6.3
13+
jupyter_core==5.7.2
14+
matplotlib-inline==0.1.7
15+
nest-asyncio==1.6.0
16+
numpy==2.2.3
17+
packaging==24.2
18+
pandas==2.2.3
19+
parso==0.8.4
20+
platformdirs==4.3.6
21+
prompt_toolkit==3.0.50
22+
psutil==7.0.0
23+
pure_eval==0.2.3
24+
Pygments==2.19.1
25+
python-dateutil==2.9.0.post0
26+
pytz==2025.1
27+
pywin32==308
28+
pyzmq==26.2.1
29+
six==1.17.0
30+
stack-data==0.6.3
31+
tornado==6.4.2
32+
traitlets==5.14.3
33+
tzdata==2025.1
34+
wcwidth==0.2.13

FOM.RAG.CrawlerOrchestrator.Abstractions/Contracts/ICrawler.cs renamed to src/FOM.RAG.CrawlerOrchestrator.Abstractions/Contracts/ICrawler.cs

File renamed without changes.

FOM.RAG.CrawlerOrchestrator.Abstractions/Contracts/IDocument.cs renamed to src/FOM.RAG.CrawlerOrchestrator.Abstractions/Contracts/IDocument.cs

File renamed without changes.

FOM.RAG.CrawlerOrchestrator.Abstractions/Dtos/Document.cs renamed to src/FOM.RAG.CrawlerOrchestrator.Abstractions/Dtos/Document.cs

File renamed without changes.

FOM.RAG.CrawlerOrchestrator.Abstractions/FOM.RAG.CrawlerOrchestrator.Abstractions.csproj renamed to src/FOM.RAG.CrawlerOrchestrator.Abstractions/FOM.RAG.CrawlerOrchestrator.Abstractions.csproj

File renamed without changes.

FOM.RAG.CrawlerOrchestrator/CrawlerOrchestratorService.cs renamed to src/FOM.RAG.CrawlerOrchestrator/CrawlerOrchestratorService.cs

File renamed without changes.

FOM.RAG.CrawlerOrchestrator/FOM.RAG.CrawlerOrchestrator.csproj renamed to src/FOM.RAG.CrawlerOrchestrator/FOM.RAG.CrawlerOrchestrator.csproj

File renamed without changes.

0 commit comments

Comments
 (0)