Skip to content

Commit b112803

Browse files
LogendeLogende
committed
decrease quality score of edges upper bound from 1 to 0.9
1 parent 392e017 commit b112803

15 files changed

Lines changed: 332 additions & 121 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,3 +224,4 @@ external_converters/java/converter.jar
224224

225225
# Isolated per-tool virtual environments (XSD2SHACL, xsd2rdf), built by scripts/build_isolated_python_tools.sh
226226
external_converters/python_isolated/
227+
/eval/results/orchestrator_outputs/runs/

‎eval/plot_orchestrator_evaluation.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ def _metrics_from_reviewed_edge_rows(group: pd.DataFrame, excluded: int = 0) ->
108108
quality_invalid = int((quality_rows["status_normalized"] == "I").sum())
109109
invalid = int((group["status_normalized"] == "I").sum())
110110
robustness = valid_outputs / cases if cases else 0.0
111-
quality = (good + 0.5 * lacking) / quality_cases if quality_cases else 0.0
111+
quality = (0.9 * good + 0.5 * lacking) / quality_cases if quality_cases else 0.0
112112
reliability = robustness * quality
113113
return {
114114
"robustness": round(robustness, 4),

‎eval/plotting_conversion_graph.py‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -68,22 +68,22 @@
6868

6969
PREFERRED_POSITIONS = {
7070
"Dtd": (-4.7, 1.8),
71-
"Xsd": (-3.1, 0.9),
71+
"Xsd": (-3.1, 1.8),
7272
"JsonSchema": (-0.8, 0.1),
7373
"LinkMl": (1.4, 1.15),
74-
"MdModels": (1.4, -1.15),
75-
"SHACL_TTL": (-1.45, -2.68),
76-
"SHACL_JSON_LD": (-2.7, -2.05),
74+
"MdModels": (0.1, -2.68),
75+
"SHACL_TTL": (-2.55, -2.68),
76+
"SHACL_JSON_LD": (-3.8, -1.35),
7777
"Owl_TTL": (0.1, 2.25),
7878
"Owl_XML": (-1.9, 2.85),
7979
"Owl_OFN": (2.0, 2.85),
8080
"OWL_OBO": (3.2, 2.0),
8181
"OntologyRdf": (-3.6, 2.85),
82-
"GraphQL": (4.0, 0.95),
83-
"Protobuf": (4.25, 0.15),
84-
"Shex": (4.1, -0.65),
85-
"SqlAlchemy": (4.05, 1.75),
86-
"Mermaid": (4.2, -1.45),
82+
"GraphQL": (4.0, 0.9),
83+
"Protobuf": (4.25, -0.1),
84+
"Shex": (4.1, -1.35),
85+
"SqlAlchemy": (4.05, 1.65),
86+
"Mermaid": (4.2, -2.45),
8787
}
8888

8989
GRAPHICAL_ABSTRACT_POSITIONS = {
@@ -430,7 +430,7 @@ def visualize_conversion_graph_with_metrics(
430430
fig, ax = plt.subplots(figsize=(13.5, 8.2))
431431
ax.set_facecolor("#FFFFFF")
432432
cmap = LinearSegmentedColormap.from_list("edge_quality", ["#C94C4C", "#F2D06B", "#5BAF72"])
433-
norm = Normalize(vmin=0.0, vmax=1.0)
433+
norm = Normalize(vmin=0.0, vmax=0.9)
434434
label_texts = []
435435

436436
for (src, tgt), edge_entries in sorted(edges_by_pair.items()):
@@ -514,8 +514,8 @@ def visualize_conversion_graph_with_metrics(
514514
sm.set_array([])
515515
cbar = fig.colorbar(sm, ax=ax, fraction=0.028, pad=0.012)
516516
cbar.set_label("Combined edge reliability: robustness x quality", fontsize=12.6)
517-
cbar.set_ticks([0.0, 1.0])
518-
cbar.set_ticklabels(["Low (0)", "High (1)"])
517+
cbar.set_ticks([0.0, 0.9])
518+
cbar.set_ticklabels(["Low (0)", "High (0.9)"])
519519
cbar.ax.tick_params(labelsize=12.6)
520520

521521
xs = [x for x, _ in pos.values()]

‎eval/plotting_conversion_matrix.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ def build_orchestrator_result_matrix(final_review: pd.DataFrame) -> tuple[pd.Dat
173173
annotations.loc[source, target] = f"{path_count}P\n{status_line}"
174174

175175
total = max(len(pair), 1)
176-
heat.loc[source, target] = (good + 0.5 * lacking) / total
176+
heat.loc[source, target] = (0.9 * good + 0.5 * lacking) / total
177177

178178
annotations.index.name = "Source Language"
179179
annotations.columns.name = "Target Language"
@@ -215,7 +215,7 @@ def _append_edge_quality_row(rows: list[dict], edge_signature: str, group: pd.Da
215215
quality_good = int((quality_rows["status_normalized"] == "G").sum())
216216
quality_lacking = int((quality_rows["status_normalized"] == "L").sum())
217217
robustness = valid_outputs / total if total else 0.0
218-
quality = (quality_good + 0.5 * quality_lacking) / quality_cases if quality_cases else 0.0
218+
quality = (0.9 * quality_good + 0.5 * quality_lacking) / quality_cases if quality_cases else 0.0
219219
score = robustness * quality
220220
rows.append({
221221
"edge_signature": edge_signature,
@@ -461,16 +461,16 @@ def plot_orchestrator_result_matrix(
461461
fmt="",
462462
cmap=cmap,
463463
vmin=0.0,
464-
vmax=1.0,
464+
vmax=0.9,
465465
linewidths=0.8,
466466
linecolor="white",
467467
cbar_kws={"label": ""},
468468
annot_kws={"fontsize": 10.8, "fontweight": "semibold"},
469469
)
470470
colorbar = axis.collections[0].colorbar
471-
colorbar.set_label("Result quality score: (G + 0.5L) / total", fontsize=12)
472-
colorbar.set_ticks([0.0, 1.0])
473-
colorbar.set_ticklabels(["Invalid (0)", "Good (1)"])
471+
colorbar.set_label("Result quality score: (0.9G + 0.5L) / total", fontsize=12)
472+
colorbar.set_ticks([0.0, 0.9])
473+
colorbar.set_ticklabels(["Invalid (0)", "Good (0.9)"])
474474
axis.set_xlabel("Output Schema Language")
475475
axis.set_ylabel("Input Schema Language")
476476
plt.xticks(rotation=45, ha="right")

‎eval/results/edge_robustness_scores.json‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
"JsonSchema:LinkMl:LinkMl schema_automator JsonSchemaImportEngine": {
33
"robustness": 1.0,
44
"validity": 1.0,
5-
"quality": 0.8333,
6-
"reliability": 0.8333,
5+
"quality": 0.7667,
6+
"reliability": 0.7667,
77
"good": 2,
88
"lacking": 1,
99
"invalid": 0,
@@ -17,8 +17,8 @@
1717
"JsonSchema:MdModels:MdModels DataModel from JsonSchema": {
1818
"robustness": 0.6667,
1919
"validity": 0.6667,
20-
"quality": 1.0,
21-
"reliability": 0.6667,
20+
"quality": 0.9,
21+
"reliability": 0.6,
2222
"good": 2,
2323
"lacking": 0,
2424
"invalid": 1,
@@ -32,8 +32,8 @@
3232
"JsonSchema:SHACL_TTL:jsonschema2shacl": {
3333
"robustness": 0.6667,
3434
"validity": 0.6667,
35-
"quality": 1.0,
36-
"reliability": 0.6667,
35+
"quality": 0.9,
36+
"reliability": 0.6,
3737
"good": 2,
3838
"lacking": 0,
3939
"invalid": 1,
@@ -47,8 +47,8 @@
4747
"JsonSchema:SHACL_TTL:shacl-bridge": {
4848
"robustness": 1.0,
4949
"validity": 1.0,
50-
"quality": 0.8333,
51-
"reliability": 0.8333,
50+
"quality": 0.7667,
51+
"reliability": 0.7667,
5252
"good": 2,
5353
"lacking": 1,
5454
"invalid": 0,
@@ -62,8 +62,8 @@
6262
"JsonSchema:Xsd:jsons2xsd": {
6363
"robustness": 0.6667,
6464
"validity": 0.6667,
65-
"quality": 0.75,
66-
"reliability": 0.5,
65+
"quality": 0.7,
66+
"reliability": 0.4667,
6767
"good": 1,
6868
"lacking": 1,
6969
"invalid": 1,
@@ -77,8 +77,8 @@
7777
"LinkMl:JsonSchema:LinkML JsonSchema Generator": {
7878
"robustness": 1.0,
7979
"validity": 1.0,
80-
"quality": 1.0,
81-
"reliability": 1.0,
80+
"quality": 0.9,
81+
"reliability": 0.9,
8282
"good": 3,
8383
"lacking": 0,
8484
"invalid": 0,
@@ -92,8 +92,8 @@
9292
"LinkMl:SHACL_TTL:LinkML SHACL_TTL Generator": {
9393
"robustness": 0.6667,
9494
"validity": 0.6667,
95-
"quality": 1.0,
96-
"reliability": 0.6667,
95+
"quality": 0.9,
96+
"reliability": 0.6,
9797
"good": 2,
9898
"lacking": 0,
9999
"invalid": 1,
@@ -107,8 +107,8 @@
107107
"MdModels:JsonSchema:MdModels Templates: JsonSchema": {
108108
"robustness": 1.0,
109109
"validity": 1.0,
110-
"quality": 1.0,
111-
"reliability": 1.0,
110+
"quality": 0.9,
111+
"reliability": 0.9,
112112
"good": 3,
113113
"lacking": 0,
114114
"invalid": 0,
@@ -137,8 +137,8 @@
137137
"MdModels:Xsd:MdModels Templates: Xsd": {
138138
"robustness": 1.0,
139139
"validity": 1.0,
140-
"quality": 1.0,
141-
"reliability": 1.0,
140+
"quality": 0.9,
141+
"reliability": 0.9,
142142
"good": 3,
143143
"lacking": 0,
144144
"invalid": 0,
@@ -152,8 +152,8 @@
152152
"SHACL_TTL:JsonSchema:shacl-bridge": {
153153
"robustness": 1.0,
154154
"validity": 1.0,
155-
"quality": 1.0,
156-
"reliability": 1.0,
155+
"quality": 0.9,
156+
"reliability": 0.9,
157157
"good": 3,
158158
"lacking": 0,
159159
"invalid": 0,
@@ -182,8 +182,8 @@
182182
"Xsd:JsonSchema:xsd-json-converter (xjc)": {
183183
"robustness": 1.0,
184184
"validity": 1.0,
185-
"quality": 0.8333,
186-
"reliability": 0.8333,
185+
"quality": 0.7667,
186+
"reliability": 0.7667,
187187
"good": 2,
188188
"lacking": 1,
189189
"invalid": 0,
@@ -197,8 +197,8 @@
197197
"Xsd:JsonSchema:xsd2jsonschema": {
198198
"robustness": 0.6667,
199199
"validity": 0.6667,
200-
"quality": 1.0,
201-
"reliability": 0.6667,
200+
"quality": 0.9,
201+
"reliability": 0.6,
202202
"good": 2,
203203
"lacking": 0,
204204
"invalid": 1,

‎eval/results/orchestrator_outputs/edge_robustness_scores.json‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
"JsonSchema:LinkMl:LinkMl schema_automator JsonSchemaImportEngine": {
33
"robustness": 1.0,
44
"validity": 1.0,
5-
"quality": 0.8333,
6-
"reliability": 0.8333,
5+
"quality": 0.7667,
6+
"reliability": 0.7667,
77
"good": 2,
88
"lacking": 1,
99
"invalid": 0,
@@ -17,8 +17,8 @@
1717
"JsonSchema:MdModels:MdModels DataModel from JsonSchema": {
1818
"robustness": 0.6667,
1919
"validity": 0.6667,
20-
"quality": 1.0,
21-
"reliability": 0.6667,
20+
"quality": 0.9,
21+
"reliability": 0.6,
2222
"good": 2,
2323
"lacking": 0,
2424
"invalid": 1,
@@ -32,8 +32,8 @@
3232
"JsonSchema:SHACL_TTL:jsonschema2shacl": {
3333
"robustness": 0.6667,
3434
"validity": 0.6667,
35-
"quality": 1.0,
36-
"reliability": 0.6667,
35+
"quality": 0.9,
36+
"reliability": 0.6,
3737
"good": 2,
3838
"lacking": 0,
3939
"invalid": 1,
@@ -47,8 +47,8 @@
4747
"JsonSchema:SHACL_TTL:shacl-bridge": {
4848
"robustness": 1.0,
4949
"validity": 1.0,
50-
"quality": 0.8333,
51-
"reliability": 0.8333,
50+
"quality": 0.7667,
51+
"reliability": 0.7667,
5252
"good": 2,
5353
"lacking": 1,
5454
"invalid": 0,
@@ -62,8 +62,8 @@
6262
"JsonSchema:Xsd:jsons2xsd": {
6363
"robustness": 0.6667,
6464
"validity": 0.6667,
65-
"quality": 0.75,
66-
"reliability": 0.5,
65+
"quality": 0.7,
66+
"reliability": 0.4667,
6767
"good": 1,
6868
"lacking": 1,
6969
"invalid": 1,
@@ -77,8 +77,8 @@
7777
"LinkMl:JsonSchema:LinkML JsonSchema Generator": {
7878
"robustness": 1.0,
7979
"validity": 1.0,
80-
"quality": 1.0,
81-
"reliability": 1.0,
80+
"quality": 0.9,
81+
"reliability": 0.9,
8282
"good": 3,
8383
"lacking": 0,
8484
"invalid": 0,
@@ -92,8 +92,8 @@
9292
"LinkMl:SHACL_TTL:LinkML SHACL_TTL Generator": {
9393
"robustness": 0.6667,
9494
"validity": 0.6667,
95-
"quality": 1.0,
96-
"reliability": 0.6667,
95+
"quality": 0.9,
96+
"reliability": 0.6,
9797
"good": 2,
9898
"lacking": 0,
9999
"invalid": 1,
@@ -107,8 +107,8 @@
107107
"MdModels:JsonSchema:MdModels Templates: JsonSchema": {
108108
"robustness": 1.0,
109109
"validity": 1.0,
110-
"quality": 1.0,
111-
"reliability": 1.0,
110+
"quality": 0.9,
111+
"reliability": 0.9,
112112
"good": 3,
113113
"lacking": 0,
114114
"invalid": 0,
@@ -137,8 +137,8 @@
137137
"MdModels:Xsd:MdModels Templates: Xsd": {
138138
"robustness": 1.0,
139139
"validity": 1.0,
140-
"quality": 1.0,
141-
"reliability": 1.0,
140+
"quality": 0.9,
141+
"reliability": 0.9,
142142
"good": 3,
143143
"lacking": 0,
144144
"invalid": 0,
@@ -152,8 +152,8 @@
152152
"SHACL_TTL:JsonSchema:shacl-bridge": {
153153
"robustness": 1.0,
154154
"validity": 1.0,
155-
"quality": 1.0,
156-
"reliability": 1.0,
155+
"quality": 0.9,
156+
"reliability": 0.9,
157157
"good": 3,
158158
"lacking": 0,
159159
"invalid": 0,
@@ -182,8 +182,8 @@
182182
"Xsd:JsonSchema:xsd-json-converter (xjc)": {
183183
"robustness": 1.0,
184184
"validity": 1.0,
185-
"quality": 0.8333,
186-
"reliability": 0.8333,
185+
"quality": 0.7667,
186+
"reliability": 0.7667,
187187
"good": 2,
188188
"lacking": 1,
189189
"invalid": 0,
@@ -197,8 +197,8 @@
197197
"Xsd:JsonSchema:xsd2jsonschema": {
198198
"robustness": 0.6667,
199199
"validity": 0.6667,
200-
"quality": 1.0,
201-
"reliability": 0.6667,
200+
"quality": 0.9,
201+
"reliability": 0.6,
202202
"good": 2,
203203
"lacking": 0,
204204
"invalid": 1,

‎external_converters/java/src/main/java/org/logende/converter/converters/DtdToXsdConverter.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,12 @@ public String convert(String schema) throws Exception {
4949

5050
File outputFile = File.createTempFile("schema", ".xsd");
5151

52-
// Call Trang programmatically
52+
// Call Trang programmatically via run() — main() calls System.exit() and kills the JVM
5353
String[] args = {inputFile.getAbsolutePath(), outputFile.getAbsolutePath()};
54-
Driver.main(args);
54+
int trangResult = new Driver().run(args);
55+
if (trangResult != 0) {
56+
throw new Exception("Trang conversion failed (exit code " + trangResult + ")");
57+
}
5558

5659
// Read result
5760
String result = new String(java.nio.file.Files.readAllBytes(outputFile.toPath()));

‎external_converters/xmlschema2shex/xmlschema2shex.jar‎

72.9 MB
Binary file not shown.

0 commit comments

Comments
 (0)