Skip to content

Commit cdfc989

Browse files
authored
Scale row collab IndexedDB cache (#336)
* feat: scale row collab IndexedDB cache * Disable database field wrap by default * chore: lint * Fix shared row legacy cache migration * chore: lint
1 parent e8e6622 commit cdfc989

32 files changed

Lines changed: 2402 additions & 177 deletions

.eslintignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,5 @@ vite.config.ts
88
*.config.ts
99
coverage/
1010
src/proto/**/*
11-
cypress/e2e/**
11+
cypress/e2e/**
12+
scripts/export-indexeddb-to-zip.js

.eslintignore.web

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,5 @@ playwright/
1313
playwright.config.ts
1414
playwright-snapshot.config.ts
1515
deploy/*.test.ts
16-
deploy/*.integration.test.ts
16+
deploy/*.integration.test.ts
17+
scripts/export-indexeddb-to-zip.js

src/application/database-blob/index.ts

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import * as Y from 'yjs';
44

55
import { hasRowConditionData, invalidateRowConditionCache } from '@/application/database-yjs/condition-value-cache';
66
import { getRowKey } from '@/application/database-yjs/row_meta';
7-
import { getCachedProviderDoc, openCollabDBWithProvider } from '@/application/db';
7+
import { getCachedProviderDoc, openCollabDBWithProvider, openRowCollabDBWithProvider } from '@/application/db';
88
import { getCachedRowDoc } from '@/application/services/js-services/cache';
99
import { databaseBlobDiff } from '@/application/services/js-services/http/http_api';
1010
import { YDoc, YjsEditorKey } from '@/application/types';
@@ -550,7 +550,11 @@ function inspectDocRowData(doc: YDoc, objectId: string): {
550550
}
551551
}
552552

553-
async function applyCollabUpdate(objectId: string, docState: database_blob.ICollabDocState) {
553+
async function applyCollabUpdate(
554+
objectId: string,
555+
docState: database_blob.ICollabDocState,
556+
options?: { useSharedRowStorage?: boolean }
557+
) {
554558
const state = getDocState(docState);
555559

556560
if (!state) {
@@ -589,7 +593,9 @@ async function applyCollabUpdate(objectId: string, docState: database_blob.IColl
589593
});
590594

591595
const openStartedAt = Date.now();
592-
const { doc, provider } = await openCollabDBWithProvider(objectId, { skipCache: true });
596+
const { doc, provider } = options?.useSharedRowStorage
597+
? await openRowCollabDBWithProvider(objectId, { skipCache: true })
598+
: await openCollabDBWithProvider(objectId, { skipCache: true });
593599

594600
const beforeState = inspectDocRowData(doc, objectId);
595601

@@ -658,7 +664,7 @@ async function applyRowUpdate(
658664
cacheRowDocSeed(rowKey, rowDocState);
659665
}
660666

661-
await applyCollabUpdate(rowKey, rowDocState);
667+
await applyCollabUpdate(rowId, rowDocState, { useSharedRowStorage: true });
662668
}
663669

664670
const doc = update.document;
Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
import { act, renderHook, waitFor } from '@testing-library/react';
2+
import * as Y from 'yjs';
3+
4+
import {
5+
DatabaseContext,
6+
DatabaseContextState,
7+
FieldType,
8+
useFieldsSelector,
9+
useFieldWrap,
10+
useTogglePropertyWrapDispatch,
11+
} from '@/application/database-yjs';
12+
import {
13+
YDatabase,
14+
YDatabaseField,
15+
YDatabaseFieldSetting,
16+
YDatabaseFieldSettings,
17+
YDatabaseView,
18+
YDoc,
19+
YjsDatabaseKey,
20+
YjsEditorKey,
21+
} from '@/application/types';
22+
23+
jest.mock('@/utils/runtime-config', () => ({
24+
getConfigValue: (_key: string, fallback: string) => fallback,
25+
}));
26+
27+
const fieldId = 'field-id';
28+
const viewId = 'view-id';
29+
30+
function createDatabaseDoc(wrap?: boolean): {
31+
databaseDoc: YDoc;
32+
fieldSettings: YDatabaseFieldSettings;
33+
} {
34+
const databaseDoc = new Y.Doc() as unknown as YDoc;
35+
const sharedRoot = databaseDoc.getMap(YjsEditorKey.data_section);
36+
const database = new Y.Map() as YDatabase;
37+
const fields = new Y.Map<YDatabaseField>();
38+
const field = new Y.Map() as YDatabaseField;
39+
const views = new Y.Map<YDatabaseView>();
40+
const view = new Y.Map() as YDatabaseView;
41+
const fieldOrders = new Y.Array<{ id: string }>();
42+
const fieldSettings = new Y.Map() as YDatabaseFieldSettings;
43+
const fieldSetting = new Y.Map() as YDatabaseFieldSetting;
44+
45+
field.set(YjsDatabaseKey.id, fieldId);
46+
field.set(YjsDatabaseKey.is_primary, true);
47+
field.set(YjsDatabaseKey.type, FieldType.RichText);
48+
fields.set(fieldId, field);
49+
50+
if (wrap !== undefined) {
51+
fieldSetting.set(YjsDatabaseKey.wrap, wrap);
52+
}
53+
54+
fieldSettings.set(fieldId, fieldSetting);
55+
fieldOrders.push([{ id: fieldId }]);
56+
view.set(YjsDatabaseKey.field_orders, fieldOrders);
57+
view.set(YjsDatabaseKey.field_settings, fieldSettings);
58+
views.set(viewId, view);
59+
60+
database.set(YjsDatabaseKey.fields, fields);
61+
database.set(YjsDatabaseKey.views, views);
62+
sharedRoot.set(YjsEditorKey.database, database);
63+
64+
return {
65+
databaseDoc,
66+
fieldSettings,
67+
};
68+
}
69+
70+
function createContextValue(databaseDoc: YDoc): DatabaseContextState {
71+
return {
72+
readOnly: false,
73+
databaseDoc,
74+
databasePageId: viewId,
75+
activeViewId: viewId,
76+
rowMap: null,
77+
workspaceId: 'workspace-id',
78+
};
79+
}
80+
81+
describe('field wrap default', () => {
82+
it('defaults selector columns to nowrap when wrap is missing', async () => {
83+
const { databaseDoc } = createDatabaseDoc();
84+
const contextValue = createContextValue(databaseDoc);
85+
86+
const { result } = renderHook(() => useFieldsSelector(), {
87+
wrapper: ({ children }) => <DatabaseContext.Provider value={contextValue}>{children}</DatabaseContext.Provider>,
88+
});
89+
90+
await waitFor(() => expect(result.current).toHaveLength(1));
91+
expect(result.current[0].wrap).toBe(false);
92+
});
93+
94+
it('defaults field wrap state to false when wrap is missing', () => {
95+
const { databaseDoc } = createDatabaseDoc();
96+
const contextValue = createContextValue(databaseDoc);
97+
98+
const { result } = renderHook(() => useFieldWrap(fieldId), {
99+
wrapper: ({ children }) => <DatabaseContext.Provider value={contextValue}>{children}</DatabaseContext.Provider>,
100+
});
101+
102+
expect(result.current).toBe(false);
103+
});
104+
105+
it('toggles a missing wrap setting from false to true', () => {
106+
const { databaseDoc, fieldSettings } = createDatabaseDoc();
107+
const contextValue = createContextValue(databaseDoc);
108+
109+
const { result } = renderHook(() => useTogglePropertyWrapDispatch(), {
110+
wrapper: ({ children }) => <DatabaseContext.Provider value={contextValue}>{children}</DatabaseContext.Provider>,
111+
});
112+
113+
act(() => {
114+
result.current(fieldId);
115+
});
116+
117+
expect(fieldSettings.get(fieldId)?.get(YjsDatabaseKey.wrap)).toBe(true);
118+
});
119+
120+
it('preserves an explicit wrap setting', async () => {
121+
const { databaseDoc } = createDatabaseDoc(true);
122+
const contextValue = createContextValue(databaseDoc);
123+
124+
const { result } = renderHook(() => useFieldsSelector(), {
125+
wrapper: ({ children }) => <DatabaseContext.Provider value={contextValue}>{children}</DatabaseContext.Provider>,
126+
});
127+
128+
await waitFor(() => expect(result.current).toHaveLength(1));
129+
expect(result.current[0].wrap).toBe(true);
130+
});
131+
});

src/application/database-yjs/const.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { RowId, YDatabaseRow, YDoc, YjsDatabaseKey, YjsEditorKey } from '@/appli
66
export const DEFAULT_ROW_HEIGHT = 36;
77
export const MIN_COLUMN_WIDTH = 150;
88
export const PADDING_END = 220;
9+
export const DEFAULT_FIELD_WRAP = false;
910

1011
export const getCell = (rowId: string, fieldId: string, rowMetas: Record<RowId, YDoc>) => {
1112
const rowMeta = rowMetas[rowId];

src/application/database-yjs/dispatch.ts

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,12 @@ import { createRollupField } from '@/application/database-yjs/fields/rollup/util
3737
import { createSelectOptionCell } from '@/application/database-yjs/fields/select-option/utils';
3838
import { createDateTimeField } from '@/application/database-yjs/fields/text/utils';
3939
import { getDefaultFilterCondition } from '@/application/database-yjs/filter';
40+
import { DEFAULT_FIELD_WRAP } from '@/application/database-yjs/const';
4041
import { getOptionsFromRow } from '@/application/database-yjs/row';
4142
import { getMetaIdMap } from '@/application/database-yjs/row_meta';
4243
import { useBoardLayoutSettings, useCalendarLayoutSetting, useFieldSelector, useFieldType } from '@/application/database-yjs/selector';
44+
import { deleteCollabDB } from '@/application/db';
45+
import { deleteOutboxByObjectId } from '@/application/sync-outbox';
4346
import { executeOperations } from '@/application/slate-yjs/utils/yjs';
4447
import {
4548
DatabaseViewLayout,
@@ -693,6 +696,8 @@ export function useDeleteRowDispatch() {
693696
},
694697
'deleteRowDispatch'
695698
);
699+
void deleteOutboxByObjectId(rowId);
700+
void deleteCollabDB(rowId, { destroyDoc: false });
696701
},
697702
[sharedRoot, database]
698703
);
@@ -729,6 +734,10 @@ export function useBulkDeleteRowDispatch() {
729734
},
730735
'bulkDeleteRowDispatch'
731736
);
737+
rowIds.forEach((rowId) => {
738+
void deleteOutboxByObjectId(rowId);
739+
void deleteCollabDB(rowId, { destroyDoc: false });
740+
});
732741
},
733742
[sharedRoot, database]
734743
);
@@ -1003,6 +1012,7 @@ export function useNewPropertyDispatch() {
10031012
const setting = new Y.Map() as YDatabaseFieldSetting;
10041013

10051014
setting.set(YjsDatabaseKey.visibility, FieldVisibility.AlwaysShown);
1015+
setting.set(YjsDatabaseKey.wrap, DEFAULT_FIELD_WRAP);
10061016
fieldSettings.set(fieldId, setting);
10071017

10081018
fieldOrders.push([
@@ -1046,6 +1056,7 @@ export function useAddPropertyLeftDispatch() {
10461056
const setting = new Y.Map() as YDatabaseFieldSetting;
10471057

10481058
setting.set(YjsDatabaseKey.visibility, FieldVisibility.AlwaysShown);
1059+
setting.set(YjsDatabaseKey.wrap, DEFAULT_FIELD_WRAP);
10491060
fieldSettings.set(newId, setting);
10501061

10511062
const index = fieldOrders.toArray().findIndex((field) => field.id === fieldId);
@@ -1092,6 +1103,7 @@ export function useAddPropertyRightDispatch() {
10921103
const setting = new Y.Map() as YDatabaseFieldSetting;
10931104

10941105
setting.set(YjsDatabaseKey.visibility, FieldVisibility.AlwaysShown);
1106+
setting.set(YjsDatabaseKey.wrap, DEFAULT_FIELD_WRAP);
10951107
fieldSettings.set(newId, setting);
10961108

10971109
const index = fieldOrders.toArray().findIndex((field) => field.id === fieldId);
@@ -1416,7 +1428,7 @@ export function useTogglePropertyWrapDispatch() {
14161428
fieldSettings.set(fieldId, setting);
14171429
}
14181430

1419-
const wrap = setting.get(YjsDatabaseKey.wrap) ?? true;
1431+
const wrap = setting.get(YjsDatabaseKey.wrap) ?? DEFAULT_FIELD_WRAP;
14201432

14211433
if (checked !== undefined) {
14221434
setting.set(YjsDatabaseKey.wrap, checked);
@@ -1881,6 +1893,7 @@ function generateBoardSetting(database: YDatabase): YDatabaseFieldSettings {
18811893
const setting = new Y.Map() as YDatabaseFieldSetting;
18821894

18831895
setting.set(YjsDatabaseKey.visibility, FieldVisibility.HideWhenEmpty);
1896+
setting.set(YjsDatabaseKey.wrap, DEFAULT_FIELD_WRAP);
18841897

18851898
fieldSettingsMap.set(id, setting);
18861899
});
@@ -2003,6 +2016,7 @@ function useEnhanceCalendarLayoutByFieldExists() {
20032016
const setting = new Y.Map() as YDatabaseFieldSetting;
20042017

20052018
setting.set(YjsDatabaseKey.visibility, FieldVisibility.AlwaysShown);
2019+
setting.set(YjsDatabaseKey.wrap, DEFAULT_FIELD_WRAP);
20062020
fieldSettings.set(fieldId, setting);
20072021
},
20082022
'newDateTimeField'

src/application/database-yjs/dispatch/row.ts

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ import {
2828
} from '@/application/database-yjs/context';
2929
import { FieldType, RowMetaKey } from '@/application/database-yjs/database.type';
3030
import { getCachedRowSubDoc } from '@/application/services/js-services/cache';
31-
import { getCachedProviderDoc, openCollabDB } from '@/application/db';
31+
import { deleteCollabDB, getCachedProviderDoc, openCollabDB } from '@/application/db';
32+
import { deleteOutboxByObjectId } from '@/application/sync-outbox';
3233
import { Log } from '@/utils/log';
3334
import { createCheckboxCell } from '@/application/database-yjs/fields/checkbox/utils';
3435
import { createSelectOptionCell } from '@/application/database-yjs/fields/select-option/utils';
@@ -246,6 +247,8 @@ export function useDeleteRowDispatch() {
246247
},
247248
'deleteRowDispatch'
248249
);
250+
void deleteOutboxByObjectId(rowId);
251+
void deleteCollabDB(rowId, { destroyDoc: false });
249252
},
250253
[sharedRoot, database]
251254
);
@@ -282,6 +285,10 @@ export function useBulkDeleteRowDispatch() {
282285
},
283286
'bulkDeleteRowDispatch'
284287
);
288+
rowIds.forEach((rowId) => {
289+
void deleteOutboxByObjectId(rowId);
290+
void deleteCollabDB(rowId, { destroyDoc: false });
291+
});
285292
},
286293
[sharedRoot, database]
287294
);

src/application/database-yjs/selector.ts

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { useCallback, useDeferredValue, useEffect, useMemo, useRef, useState } f
55
import { parseYDatabaseCellToCell } from '@/application/database-yjs/cell.parse';
66
import { DateTimeCell, RollupCell } from '@/application/database-yjs/cell.type';
77
import { hasRowConditionData, invalidateRowConditionCache } from '@/application/database-yjs/condition-value-cache';
8-
import { getCell, MIN_COLUMN_WIDTH } from '@/application/database-yjs/const';
8+
import { DEFAULT_FIELD_WRAP, getCell, MIN_COLUMN_WIDTH } from '@/application/database-yjs/const';
99
import {
1010
useDatabase,
1111
useDatabaseContext,
@@ -86,12 +86,16 @@ function shouldLogDatabaseConditionPerformance() {
8686
return typeof window !== 'undefined' && window.location.hostname === 'localhost';
8787
}
8888

89+
function stringifyConditionSignature(value: unknown) {
90+
return JSON.stringify(value, (_key, item) => (typeof item === 'bigint' ? item.toString() : item));
91+
}
92+
8993
function getConditionSignature(sorts?: YDatabaseSorts, filters?: YDatabaseFilters) {
9094
const hasConditions = (sorts?.length ?? 0) > 0 || (filters?.length ?? 0) > 0;
9195

9296
if (!hasConditions) return '';
9397

94-
return JSON.stringify({
98+
return stringifyConditionSignature({
9599
filters: filters?.toJSON?.() ?? [],
96100
sorts: sorts?.toJSON?.() ?? [],
97101
});
@@ -266,7 +270,7 @@ export function useFieldsSelector(visibilitys: FieldVisibility[] = defaultVisibl
266270
visibility: Number(
267271
setting?.get(YjsDatabaseKey.visibility) || FieldVisibility.AlwaysShown
268272
) as FieldVisibility,
269-
wrap: setting?.get(YjsDatabaseKey.wrap) ?? true,
273+
wrap: setting?.get(YjsDatabaseKey.wrap) ?? DEFAULT_FIELD_WRAP,
270274
fieldType: Number(field?.get(YjsDatabaseKey.type)) as FieldType,
271275
};
272276
})
@@ -351,13 +355,13 @@ export function useFieldWrap(fieldId: string) {
351355
const fieldSettings = view?.get(YjsDatabaseKey.field_settings);
352356
const fieldSetting = fieldSettings?.get(fieldId);
353357

354-
const [wrap, setWrap] = useState(fieldSetting?.get(YjsDatabaseKey.wrap) ?? true);
358+
const [wrap, setWrap] = useState(fieldSetting?.get(YjsDatabaseKey.wrap) ?? DEFAULT_FIELD_WRAP);
355359

356360
useEffect(() => {
357361
if (!view) return;
358362

359363
const observerEvent = () => {
360-
setWrap(fieldSetting?.get(YjsDatabaseKey.wrap) ?? true);
364+
setWrap(fieldSetting?.get(YjsDatabaseKey.wrap) ?? DEFAULT_FIELD_WRAP);
361365
};
362366

363367
observerEvent();

0 commit comments

Comments
 (0)