Skip to content

Commit a3a93a3

Browse files
pyphiliakim
pyphilia
and
kim
authored
feat: optimize query for indexing items (#1937)
* feat: optimize query for indexing items * refactor: fix tests * refactor: fix tests * refactor: apply PR requested changes --------- Co-authored-by: kim <kim.phanhoang@epfl.ch>
1 parent 9e71d30 commit a3a93a3

5 files changed

Lines changed: 419 additions & 441 deletions

File tree

‎src/services/item/plugins/publication/published/plugins/search/meilisearch.repository.test.ts‎

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
import { faker } from '@faker-js/faker';
2+
3+
import { seedFromJson } from '../../../../../../../../test/mocks/seed';
4+
import { db } from '../../../../../../../drizzle/db';
5+
import { ItemType } from '../../../../../../../drizzle/types';
6+
import { assertIsDefined } from '../../../../../../../utils/assertions';
7+
import { TagCategory } from '../../../../../../tag/tag.schemas';
8+
import { MeilisearchRepository } from './meilisearch.repository';
9+
10+
const meilisearchRepository = new MeilisearchRepository();
11+
12+
describe('Meilisearch Repository', () => {
13+
describe('getIndexedTree', () => {
14+
it('return correct items', async () => {
15+
const { items, members, publishedItems, tags } = await seedFromJson({
16+
actor: null,
17+
items: [
18+
{
19+
isPublic: true,
20+
isPublished: true,
21+
creator: { name: 'cedric' },
22+
name: 'parent',
23+
description: 'parent description',
24+
tags: [{ category: TagCategory.Level, name: faker.word.words(5) }],
25+
children: [
26+
{
27+
name: 'document',
28+
type: ItemType.DOCUMENT,
29+
description: 'document description with some <div>html</div>',
30+
extra: { document: { content: 'document content with some <div>html</div>' } },
31+
},
32+
{
33+
name: 'file',
34+
description: null,
35+
type: ItemType.FILE,
36+
extra: { file: { content: 'pdf content' } },
37+
},
38+
{ extra: { folder: {} }, type: 'folder', likes: [{ name: 'bob' }, { name: 'anna' }] },
39+
{
40+
extra: { folder: {} },
41+
type: 'folder',
42+
tags: [
43+
{ category: TagCategory.Discipline, name: faker.word.words(5) },
44+
{ category: TagCategory.Discipline, name: faker.word.words(5) },
45+
],
46+
},
47+
],
48+
},
49+
],
50+
});
51+
assertIsDefined(publishedItems);
52+
53+
const results = await meilisearchRepository.getIndexedTree(db, items[0].path);
54+
55+
// parent item
56+
const parent = results.find(({ id }) => id === items[0].id);
57+
expect(parent).toEqual({
58+
id: items[0].id,
59+
name: 'parent',
60+
description: 'parent description',
61+
type: ItemType.FOLDER,
62+
lang: items[0].lang,
63+
content: '',
64+
level: [tags[0].name],
65+
discipline: [],
66+
'resource-type': [],
67+
creator: { id: members[2].id, name: members[2].name },
68+
updatedAt: items[0].updatedAt,
69+
createdAt: items[0].createdAt,
70+
isPublishedRoot: true,
71+
publicationUpdatedAt: publishedItems[0].updatedAt,
72+
isHidden: false,
73+
likes: 0,
74+
});
75+
76+
// document with content, remove html
77+
const document = results.find(({ id }) => id === items[1].id);
78+
expect(document).toEqual({
79+
id: items[1].id,
80+
name: 'document',
81+
description: 'document description with some html',
82+
type: ItemType.DOCUMENT,
83+
content: 'document content with some html',
84+
lang: items[1].lang,
85+
level: [],
86+
discipline: [],
87+
'resource-type': [],
88+
creator: { id: '', name: '' },
89+
updatedAt: items[1].updatedAt,
90+
createdAt: items[1].createdAt,
91+
isPublishedRoot: false,
92+
publicationUpdatedAt: publishedItems[0].updatedAt,
93+
isHidden: false,
94+
likes: 0,
95+
});
96+
97+
// file with content
98+
const file = results.find(({ id }) => id === items[2].id);
99+
expect(file).toEqual({
100+
id: items[2].id,
101+
name: 'file',
102+
description: '',
103+
type: ItemType.FILE,
104+
content: 'pdf content',
105+
lang: items[2].lang,
106+
level: [],
107+
discipline: [],
108+
'resource-type': [],
109+
creator: { id: '', name: '' },
110+
updatedAt: items[2].updatedAt,
111+
createdAt: items[2].createdAt,
112+
isPublishedRoot: false,
113+
publicationUpdatedAt: publishedItems[0].updatedAt,
114+
isHidden: false,
115+
likes: 0,
116+
});
117+
118+
// item with likes
119+
const likedItem = results.find(({ id }) => id === items[3].id);
120+
expect(likedItem).toEqual({
121+
id: items[3].id,
122+
name: items[3].name,
123+
description: items[3].description,
124+
type: items[3].type,
125+
content: '',
126+
lang: items[3].lang,
127+
level: [],
128+
discipline: [],
129+
'resource-type': [],
130+
creator: { id: '', name: '' },
131+
updatedAt: items[3].updatedAt,
132+
createdAt: items[3].createdAt,
133+
isPublishedRoot: false,
134+
publicationUpdatedAt: publishedItems[0].updatedAt,
135+
isHidden: false,
136+
likes: 2,
137+
});
138+
139+
// item with 2 tags
140+
const itemWith2Tags = results.find(({ id }) => id === items[4].id);
141+
expect(itemWith2Tags).toEqual({
142+
id: items[4].id,
143+
name: items[4].name,
144+
description: items[4].description,
145+
type: items[4].type,
146+
content: '',
147+
lang: items[4].lang,
148+
level: [],
149+
discipline: expect.arrayContaining([tags[1].name, tags[2].name]),
150+
'resource-type': [],
151+
creator: { id: '', name: '' },
152+
updatedAt: items[4].updatedAt,
153+
createdAt: items[4].createdAt,
154+
isPublishedRoot: false,
155+
publicationUpdatedAt: publishedItems[0].updatedAt,
156+
isHidden: false,
157+
likes: 0,
158+
});
159+
});
160+
it('return nothing for non published item', async () => {
161+
const { items } = await seedFromJson({
162+
actor: null,
163+
items: [
164+
{
165+
children: [{}],
166+
},
167+
],
168+
});
169+
170+
const result = await meilisearchRepository.getIndexedTree(db, items[0].path);
171+
172+
expect(result).toHaveLength(0);
173+
});
174+
it('return nothing for deleted item', async () => {
175+
const { items } = await seedFromJson({
176+
actor: null,
177+
items: [
178+
{
179+
isPublic: true,
180+
isPublished: true,
181+
isDeleted: true,
182+
children: [
183+
{
184+
isDeleted: true,
185+
},
186+
],
187+
},
188+
],
189+
});
190+
191+
const result = await meilisearchRepository.getIndexedTree(db, items[0].path);
192+
193+
expect(result).toHaveLength(0);
194+
});
195+
it('return nothing for hidden item', async () => {
196+
const { items } = await seedFromJson({
197+
actor: null,
198+
items: [
199+
{
200+
isPublic: true,
201+
isPublished: true,
202+
isHidden: true,
203+
children: [{}],
204+
},
205+
],
206+
});
207+
208+
const result = await meilisearchRepository.getIndexedTree(db, items[0].path);
209+
210+
expect(result).toHaveLength(0);
211+
});
212+
it('return only parent if child item is hidden', async () => {
213+
const { items } = await seedFromJson({
214+
actor: null,
215+
items: [
216+
{
217+
isPublic: true,
218+
isPublished: true,
219+
children: [
220+
{
221+
isHidden: true,
222+
},
223+
],
224+
},
225+
],
226+
});
227+
228+
const result = await meilisearchRepository.getIndexedTree(db, items[0].path);
229+
230+
expect(result).toHaveLength(1);
231+
expect(result[0].id).toEqual(items[0].id);
232+
});
233+
});
234+
});

‎src/services/item/plugins/publication/published/plugins/search/meilisearch.repository.ts‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
import { and, count, eq, isNull, sql } from 'drizzle-orm';
2+
import { singleton } from 'tsyringe';
3+
4+
import { type IndexItem, ItemVisibilityType } from '@graasp/sdk';
5+
6+
import { DBConnection } from '../../../../../../../drizzle/db';
7+
import { isAncestorOrSelf, isDescendantOrSelf } from '../../../../../../../drizzle/operations';
8+
import {
9+
accountsTable,
10+
itemLikesTable,
11+
itemTagsTable,
12+
itemVisibilitiesTable,
13+
items,
14+
publishedItemsTable,
15+
tagsTable,
16+
} from '../../../../../../../drizzle/schema';
17+
import { ItemRaw } from '../../../../../../../drizzle/types';
18+
import { TagCategory } from '../../../../../../tag/tag.schemas';
19+
import { stripHtml } from '../../../validation/utils';
20+
21+
@singleton()
22+
export class MeilisearchRepository {
23+
constructor() {}
24+
25+
public async getIndexedTree(db: DBConnection, itemPath: ItemRaw['path']): Promise<IndexItem[]> {
26+
// Define CTE "tree" with descendants and self of given item that are not hidden
27+
const tree = db.$with('tree').as(
28+
db
29+
.select({
30+
id: items.id,
31+
name: items.name,
32+
description: items.description,
33+
creatorId: items.creatorId,
34+
path: items.path,
35+
type: items.type,
36+
lang: items.lang,
37+
createdAt: items.createdAt,
38+
updatedAt: items.updatedAt,
39+
extra: items.extra,
40+
})
41+
.from(items)
42+
.leftJoin(
43+
itemVisibilitiesTable,
44+
and(
45+
isAncestorOrSelf(itemVisibilitiesTable.itemPath, items.path),
46+
eq(itemVisibilitiesTable.type, ItemVisibilityType.Hidden),
47+
),
48+
)
49+
.where(and(isDescendantOrSelf(items.path, itemPath), isNull(itemVisibilitiesTable.id))),
50+
);
51+
52+
// Define "tags" CTE with filtered aggregation for each tag category
53+
const tagsCte = db.$with('tags').as(
54+
db
55+
.select({
56+
itemId: itemTagsTable.itemId,
57+
discipline: sql<
58+
string[]
59+
>`array_agg(${tagsTable.name}) filter (where ${tagsTable.category}=${TagCategory.Discipline})`.as(
60+
'discipline',
61+
),
62+
level: sql<
63+
string[]
64+
>`array_agg(${tagsTable.name}) filter (where ${tagsTable.category}=${TagCategory.Level})`.as(
65+
'level',
66+
),
67+
resourceType: sql<
68+
string[]
69+
>`array_agg(${tagsTable.name}) filter (where ${tagsTable.category}=${TagCategory.ResourceType})`.as(
70+
'resourceType',
71+
),
72+
})
73+
.from(tagsTable)
74+
.leftJoin(itemTagsTable, eq(tagsTable.id, itemTagsTable.tagId))
75+
.innerJoin(tree, eq(tree.id, itemTagsTable.itemId))
76+
.groupBy(itemTagsTable.itemId),
77+
);
78+
79+
// Define "likes" CTE counting likes per item
80+
const likesCte = db.$with('likes').as(
81+
db
82+
.select({
83+
itemId: itemLikesTable.itemId,
84+
count: count(itemLikesTable.itemId).as('count'),
85+
})
86+
.from(itemLikesTable)
87+
.innerJoin(tree, eq(tree.id, itemLikesTable.itemId))
88+
.groupBy(itemLikesTable.itemId),
89+
);
90+
91+
// Build main query referencing the CTEs and joins
92+
const result = await db
93+
.with(tree, tagsCte, likesCte)
94+
.select({
95+
// item properties
96+
id: tree.id,
97+
name: tree.name,
98+
description: tree.description,
99+
path: tree.path,
100+
type: tree.type,
101+
// get content for documents and files
102+
content: sql<string>`COALESCE(${tree.extra}->'document'->>'content', ${tree.extra}->'file'->>'content','')`,
103+
lang: tree.lang,
104+
createdAt: tree.createdAt,
105+
updatedAt: tree.updatedAt,
106+
// publication
107+
publishedRootPath: publishedItemsTable.itemPath,
108+
publicationUpdatedAt: publishedItemsTable.updatedAt,
109+
// tags
110+
// use coalesce to return an empty array if the item does not have tags
111+
discipline: sql<string[]>`COALESCE(${tagsCte.discipline},'{}')`,
112+
level: sql<string[]>`COALESCE(${tagsCte.level},'{}')`,
113+
resourceType: sql<string[]>`COALESCE(${tagsCte.resourceType},'{}')`,
114+
// creator
115+
creatorId: accountsTable.id,
116+
creatorName: accountsTable.name,
117+
// likes
118+
// coalesce in case item does not have any like
119+
likes: sql<number>`COALESCE(${likesCte.count},0)::int`,
120+
})
121+
.from(tree)
122+
.innerJoin(publishedItemsTable, isAncestorOrSelf(publishedItemsTable.itemPath, tree.path))
123+
.leftJoin(accountsTable, eq(accountsTable.id, tree.creatorId))
124+
.leftJoin(tagsCte, eq(tagsCte.itemId, tree.id))
125+
.leftJoin(likesCte, eq(likesCte.itemId, tree.id));
126+
127+
return result.map((doc) => ({
128+
id: doc.id,
129+
name: doc.name,
130+
creator: {
131+
id: doc.creatorId ?? '',
132+
name: doc.creatorName ?? '',
133+
},
134+
description: doc.description ? stripHtml(doc.description) : '',
135+
type: doc.type,
136+
content: doc.content ? stripHtml(doc.content) : '',
137+
isPublishedRoot: doc.path === doc.publishedRootPath,
138+
publicationUpdatedAt: doc.publicationUpdatedAt,
139+
// to be removed
140+
isHidden: false,
141+
createdAt: doc.createdAt,
142+
updatedAt: doc.updatedAt,
143+
lang: doc.lang,
144+
likes: doc.likes,
145+
discipline: doc.discipline,
146+
level: doc.level,
147+
'resource-type': doc.resourceType,
148+
}));
149+
}
150+
}

0 commit comments

Comments
 (0)