Skip to content

Commit 0a64a9c

Browse files
apdavisonapdavison
committed
For curator users, merge IN_PROGRESS and RELEASED KG query results
IN_PROGRESS can return less complete data than RELEASED (e.g. empty cellPatching arrays). When auth.isCurator is true, route loaders now pass stage=["IN_PROGRESS","RELEASED"]; getKGItem and getKGData fetch both in parallel and deep-merge, with IN_PROGRESS taking precedence for non-empty fields and RELEASED filling gaps. Also fixes a bug in patchClampRecordings.jsx where stage was being passed as searchFilters due to a missing argument.
1 parent f86f3af commit 0a64a9c

9 files changed

Lines changed: 191 additions & 39 deletions

File tree

‎apps/nar-v3/__tests__/fixtures/kg_result_in_progress.json‎

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

‎apps/nar-v3/__tests__/fixtures/kg_result_released.json‎

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

‎apps/nar-v3/__tests__/utility.test.js‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,64 @@
1-
import { expect, test } from "vitest";
2-
import { getKGSearchUrl } from "../src/utility";
1+
import { describe, expect, test } from "vitest";
2+
import { getKGSearchUrl, mergeItems } from "../src/utility";
3+
import inProgressResult from "./fixtures/kg_result_in_progress.json";
4+
import releasedResult from "./fixtures/kg_result_released.json";
35

46
test("generates EBRAINS Search URL as expected", () => {
57
expect(
68
getKGSearchUrl("https://kg.ebrains.eu/api/instances/2843990a-69dd-468b-a1d3-ff9589b485ae")
79
).toBe("https://search.kg.ebrains.eu/instances/2843990a-69dd-468b-a1d3-ff9589b485ae");
810
});
11+
12+
describe("mergeItems", () => {
13+
const inProgress = inProgressResult.data[0];
14+
const released = releasedResult.data[0];
15+
const merged = mergeItems(inProgress, released);
16+
17+
test("preserves top-level fields from IN_PROGRESS", () => {
18+
expect(merged.id).toBe(inProgress.id);
19+
expect(merged.shortName).toBe(inProgress.shortName);
20+
expect(merged.versionIdentifier).toBe(inProgress.versionIdentifier);
21+
});
22+
23+
test("preserves all studiedSpecimen entries", () => {
24+
expect(merged.studiedSpecimen).toHaveLength(inProgress.studiedSpecimen.length);
25+
});
26+
27+
test("each studiedSpecimen has studiedState", () => {
28+
for (const specimen of merged.studiedSpecimen) {
29+
expect(specimen.studiedState.length).toBeGreaterThan(0);
30+
}
31+
});
32+
33+
test("each studiedState has slicePreparation", () => {
34+
for (const specimen of merged.studiedSpecimen) {
35+
for (const state of specimen.studiedState) {
36+
expect(state.slicePreparation.length).toBeGreaterThan(0);
37+
}
38+
}
39+
});
40+
41+
test("preserves string values in type arrays", () => {
42+
// type arrays are arrays of URL strings; merging must not corrupt them into
43+
// character-indexed objects via object spread of a string
44+
const slicePrep =
45+
merged.studiedSpecimen[0].studiedState[0].slicePreparation[0];
46+
expect(typeof slicePrep.type[0]).toBe("string");
47+
});
48+
49+
test("fills empty cellPatching arrays from RELEASED", () => {
50+
// IN_PROGRESS has cellPatching=[] for all outputs; RELEASED has cellPatching=[1 item]
51+
for (const specimen of merged.studiedSpecimen) {
52+
for (const state of specimen.studiedState) {
53+
for (const slicePrep of state.slicePreparation) {
54+
for (const output of slicePrep.output) {
55+
expect(
56+
output.cellPatching.length,
57+
`cellPatching empty for output: ${output.lookupLabel}`
58+
).toBeGreaterThan(0);
59+
}
60+
}
61+
}
62+
}
63+
});
64+
});

‎apps/nar-v3/src/datastore.js‎

Lines changed: 49 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ limitations under the License.
1919
*/
2020

2121
import { kgUrl, kgDefaultStage } from "./globals";
22+
import { mergeItems } from "./utility";
2223
//import examplePatchClampData from "./example_data/example_patch_clamp_dataset.json";
2324

2425
function isEmpty(obj) {
@@ -73,11 +74,24 @@ async function queryKG(kgQuery, searchParams, auth) {
7374
async function getKGItem(cacheLabel, kgQuery, instanceId, auth, stage = kgDefaultStage) {
7475
console.log("getKGItem " + cacheLabel + instanceId);
7576
if (!cache[cacheLabel][instanceId]) {
76-
const searchParams = { stage: stage, instanceId: instanceId };
77-
const result = await queryKG(kgQuery, searchParams, auth);
78-
if (result) {
79-
const items = result.data;
80-
cache[cacheLabel][instanceId] = items[0];
77+
if (Array.isArray(stage)) {
78+
// Curators pass stage as an array (e.g. ["IN_PROGRESS", "RELEASED"]) so that
79+
// we can fetch the same item under both stages in parallel and merge the results.
80+
// This is necessary because IN_PROGRESS sometimes returns less complete data
81+
// than RELEASED — for example, nested arrays that are empty in IN_PROGRESS but
82+
// populated in RELEASED. mergeItems fills those gaps while keeping IN_PROGRESS
83+
// values wherever they are present, so curators see the union of both.
84+
const results = await Promise.all(
85+
stage.map((s) => queryKG(kgQuery, { stage: s, instanceId }, auth))
86+
);
87+
const items = results.map((r) => r?.data?.[0]);
88+
cache[cacheLabel][instanceId] = mergeItems(items[0], items[1]);
89+
} else {
90+
const searchParams = { stage: stage, instanceId: instanceId };
91+
const result = await queryKG(kgQuery, searchParams, auth);
92+
if (result) {
93+
cache[cacheLabel][instanceId] = result.data[0];
94+
}
8195
}
8296
}
8397
return cache[cacheLabel][instanceId];
@@ -96,20 +110,36 @@ async function getKGData(
96110
if (isEmpty(cache[cacheLabel])) {
97111
// if the cache is empty we need to fill it
98112
console.log(kgUrl);
99-
let searchParams = {
100-
returnTotalResults: true,
101-
stage: stage,
102-
size: size,
103-
from: from,
104-
};
105-
if (searchFilters) {
106-
searchParams = { ...searchParams, searchFilters };
107-
}
108-
const result = await queryKG(kgQuery, searchParams, auth);
109-
if (result) {
110-
const items = result.data;
111-
for (const index in items) {
112-
cache[cacheLabel][items[index].id] = items[index];
113+
if (Array.isArray(stage)) {
114+
const fetchForStage = async (s) => {
115+
let searchParams = { returnTotalResults: true, stage: s, size, from };
116+
if (searchFilters) searchParams = { ...searchParams, searchFilters };
117+
const result = await queryKG(kgQuery, searchParams, auth);
118+
return result ? result.data : [];
119+
};
120+
const [primaryItems, fallbackItems] = await Promise.all(stage.map(fetchForStage));
121+
const primaryById = Object.fromEntries(primaryItems.map((i) => [i.id, i]));
122+
const fallbackById = Object.fromEntries(fallbackItems.map((i) => [i.id, i]));
123+
const allIds = new Set([...Object.keys(primaryById), ...Object.keys(fallbackById)]);
124+
for (const id of allIds) {
125+
cache[cacheLabel][id] = mergeItems(primaryById[id], fallbackById[id]);
126+
}
127+
} else {
128+
let searchParams = {
129+
returnTotalResults: true,
130+
stage: stage,
131+
size: size,
132+
from: from,
133+
};
134+
if (searchFilters) {
135+
searchParams = { ...searchParams, searchFilters };
136+
}
137+
const result = await queryKG(kgQuery, searchParams, auth);
138+
if (result) {
139+
const items = result.data;
140+
for (const index in items) {
141+
cache[cacheLabel][items[index].id] = items[index];
142+
}
113143
}
114144
}
115145
}

‎apps/nar-v3/src/routes/dataset.jsx‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,7 @@ import { basicDatasetQuery, patchClampDatasetQuery, techniquesQuery } from "./qu
3131

3232
export function getLoader(auth) {
3333
const loader = async ({ params }) => {
34-
let stage = "RELEASED";
35-
if (auth.isCurator) {
36-
stage = "IN_PROGRESS";
37-
}
34+
const stage = auth.isCurator ? ["IN_PROGRESS", "RELEASED"] : "RELEASED";
3835
const techniques = await getKGItem(
3936
"datasets techniques",
4037
techniquesQuery,

‎apps/nar-v3/src/routes/datasets.jsx‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,7 @@ import { ephysDatasetsQuery } from "./queryLibrary";
3030

3131
export function getLoader(auth) {
3232
const loader = async () => {
33-
let stage = "RELEASED";
34-
if (auth.isCurator) {
35-
stage = "IN_PROGRESS";
36-
}
33+
const stage = auth.isCurator ? ["IN_PROGRESS", "RELEASED"] : "RELEASED";
3734
const datasetsPromise = getKGData("datasets summary", ephysDatasetsQuery, auth, {}, stage);
3835
console.log(datasetsPromise);
3936
return defer({ datasets: datasetsPromise });

‎apps/nar-v3/src/routes/patchClampRecording.jsx‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -91,10 +91,7 @@ const query = buildKGQuery("TissueSample", [
9191

9292
export function getLoader(auth) {
9393
const loader = async ({ params }) => {
94-
let stage = "RELEASED";
95-
if (auth.isCurator) {
96-
stage = "IN_PROGRESS";
97-
}
94+
const stage = auth.isCurator ? ["IN_PROGRESS", "RELEASED"] : "RELEASED";
9895
const tissueSamplePromise = getKGItem(
9996
"patch clamp recordings detail",
10097
query,

‎apps/nar-v3/src/routes/patchClampRecordings.jsx‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,8 @@ export const query = buildKGQuery("TissueSample", [
4848

4949
export function getLoader(auth) {
5050
const loader = async () => {
51-
let stage = "RELEASED";
52-
if (auth.isCurator) {
53-
stage = "IN_PROGRESS";
54-
}
55-
const tissueSamplesPromise = getKGData("patch clamp recordings summary", query, auth, stage);
51+
const stage = auth.isCurator ? ["IN_PROGRESS", "RELEASED"] : "RELEASED";
52+
const tissueSamplesPromise = getKGData("patch clamp recordings summary", query, auth, {}, stage);
5653

5754
console.log(tissueSamplesPromise);
5855
return defer({ tissueSamples: tissueSamplesPromise });

‎apps/nar-v3/src/utility.js‎

Lines changed: 77 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,4 +78,80 @@ function getKGSearchUrl(uri) {
7878
return `https://search.kg.ebrains.eu/instances/${uuid}`;
7979
}
8080

81-
export { formatQuant, formatUnits, formatSolution, uuidFromUri, getKGSearchUrl };
81+
function isEmptyValue(val) {
82+
if (val === null || val === undefined || val === "") return true;
83+
if (Array.isArray(val)) return val.length === 0;
84+
if (typeof val === "object") return Object.keys(val).length === 0;
85+
return false;
86+
}
87+
88+
function getMatchKey(item) {
89+
if (!item || typeof item !== "object") return null;
90+
return item.id || item.lookupLabel || item.internalIdentifier || null;
91+
}
92+
93+
function mergeArrays(primary, fallback) {
94+
// Merge two arrays by matching elements on id, lookupLabel, or internalIdentifier.
95+
// Falls back to positional matching for items without a key.
96+
if (!primary || primary.length === 0) return fallback || [];
97+
if (!fallback || fallback.length === 0) return primary;
98+
99+
const fallbackByKey = {};
100+
const fallbackPositional = [];
101+
for (const item of fallback) {
102+
const key = getMatchKey(item);
103+
if (key) {
104+
fallbackByKey[key] = item;
105+
} else {
106+
fallbackPositional.push(item);
107+
}
108+
}
109+
110+
let positionalIndex = 0;
111+
const result = primary.map((item) => {
112+
const key = getMatchKey(item);
113+
if (key && fallbackByKey[key]) {
114+
return mergeItems(item, fallbackByKey[key]);
115+
} else if (!key && fallbackPositional[positionalIndex]) {
116+
return mergeItems(item, fallbackPositional[positionalIndex++]);
117+
}
118+
return item;
119+
});
120+
121+
// Add fallback items whose key wasn't present in primary
122+
const primaryKeys = new Set(primary.map(getMatchKey).filter(Boolean));
123+
for (const item of fallback) {
124+
const key = getMatchKey(item);
125+
if (key && !primaryKeys.has(key)) {
126+
result.push(item);
127+
}
128+
}
129+
130+
return result;
131+
}
132+
133+
function mergeItems(primary, fallback) {
134+
// Deep-merge two KG items. Primary (IN_PROGRESS) wins for non-empty fields;
135+
// empty primary fields are filled from fallback (RELEASED).
136+
if (primary === null || primary === undefined) return fallback;
137+
if (typeof primary !== "object") return primary;
138+
if (fallback === null || fallback === undefined || typeof fallback !== "object") return primary;
139+
const result = { ...primary };
140+
for (const key of Object.keys(fallback)) {
141+
if (isEmptyValue(result[key])) {
142+
result[key] = fallback[key];
143+
} else if (Array.isArray(result[key]) && Array.isArray(fallback[key])) {
144+
result[key] = mergeArrays(result[key], fallback[key]);
145+
} else if (
146+
typeof result[key] === "object" &&
147+
!Array.isArray(result[key]) &&
148+
typeof fallback[key] === "object" &&
149+
!Array.isArray(fallback[key])
150+
) {
151+
result[key] = mergeItems(result[key], fallback[key]);
152+
}
153+
}
154+
return result;
155+
}
156+
157+
export { formatQuant, formatUnits, formatSolution, uuidFromUri, getKGSearchUrl, isEmptyValue, mergeItems };

0 commit comments

Comments
 (0)