Skip to content

Commit fd1cd01

Browse files
author
djinni-hppro
committed
bulk sync improve
1 parent af07205 commit fd1cd01

4 files changed

Lines changed: 238 additions & 18 deletions

File tree

‎docs/bulk_sync_improvements.md‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
# Feature Plan: Bulk Sync Improvements
2+
3+
We are implementing two specific sync enhancements requested by the user:
4+
1. **Item 2: Guard Upload Selection on Failed Listings:** If a sync provider's listing fails, we must not falsely assume our local notes are missing from that provider (which would trigger unnecessary massive re-upload storms). We will track which provider listings failed and bypass the "missing" check for those failed providers.
5+
2. **Item 3: S3 Bulk Delete:** S3 supports deleting up to 1,000 keys in a single batch request (`deleteObjects`). We will implement `deleteNotesFromS3Bulk` in `src/s3.js` and integrate it into `helpers.js:synchronize` to execute deletes in a single batched S3 request.
6+
7+
---
8+
9+
## 1. Step-by-Step Design for Item 2 (Guard Upload Selection on Failed Listings)
10+
11+
### 1.1 Update `listNotes()` in `src/helpers.js`
12+
Modify `listNotes` to return `failedProviders` listing status:
13+
- Return `failedProviders` object mapping each provider key to a boolean status indicating whether the listing failed.
14+
- A provider is considered failed if its settled promise status is `'rejected'`.
15+
16+
```javascript
17+
// Inside listNotes()
18+
const failedProviders = {
19+
s3: s3Res.status === 'rejected',
20+
nostr: nostrRes.status === 'rejected',
21+
git: gitRes.status === 'rejected',
22+
gdrive: gdriveRes.status === 'rejected',
23+
ipfs: ipfsRes.status === 'rejected',
24+
};
25+
26+
return {
27+
mergedNotes: Array.from(mergedNotes.values()),
28+
s3Ids: new Set(s3Notes.map(n => n.id)),
29+
gitIds: new Set(gitNotes.map(n => n.id)),
30+
// ... maps and sets
31+
listingSucceeded,
32+
failedProviders
33+
};
34+
```
35+
36+
### 1.2 Update `synchronize()` upload decision filter in `src/helpers.js`
37+
In `synchronize()`, extract `failedProviders` from the result of `listNotes()`.
38+
When determining `notesToUpload`, check `!failedProviders.<provider_key>` before determining if a note is missing from that provider:
39+
40+
```javascript
41+
const { mergedNotes: remoteNoteMetadata, s3Ids, ..., failedProviders } = await listNotes(...);
42+
43+
// Inside notesToUpload calculation:
44+
if (gitCredentials?.repoUrl && !failedProviders.git) {
45+
const gitMeta = gitMap.get(localNote.id);
46+
if (!gitMeta || localDate > new Date(gitMeta.updatedAt || gitMeta.createdAt)) return true;
47+
}
48+
if (credentials.secretAccessKey && !failedProviders.s3) {
49+
const s3Meta = s3Map.get(localNote.id);
50+
if (!s3Meta || localDate > new Date(s3Meta.updatedAt)) return true;
51+
}
52+
if (nostrPrivateKey && !failedProviders.nostr) {
53+
const nostrMeta = nostrMap.get(localNote.id);
54+
if (!nostrMeta || localDate > new Date(nostrMeta.updatedAt || nostrMeta.createdAt)) return true;
55+
}
56+
if (gdriveStore?.connected && !failedProviders.gdrive) {
57+
const gdriveMeta = gdriveMap.get(localNote.id);
58+
if (!gdriveMeta || localDate > new Date(gdriveMeta.updatedAt || gdriveMeta.createdAt)) return true;
59+
}
60+
if ((credentials.pinataJwt || credentials.pinataApiKey || credentials.useDirectIpfs) && !failedProviders.ipfs) {
61+
const ipfsMeta = ipfsMap.get(localNote.id);
62+
if (!ipfsMeta || localDate > new Date(ipfsMeta.updatedAt)) return true;
63+
}
64+
```
65+
66+
---
67+
68+
## 2. Step-by-Step Design for Item 3 (S3 Bulk Delete)
69+
70+
### 2.1 Implement `deleteNotesFromS3Bulk` in `src/s3.js`
71+
Implement a new function `deleteNotesFromS3Bulk` that aggregates keys and calls `s3.deleteObjects()` in a single request.
72+
- Ensure flat keys (for backward compatibility) are included in the deletion batch.
73+
- Export `deleteNotesFromS3Bulk` on `_GLOBAL`.
74+
75+
```javascript
76+
const deleteNotesFromS3Bulk = async (notesOrIds, creds) => {
77+
if (!creds?.secretAccessKey || !notesOrIds || notesOrIds.length === 0) return;
78+
79+
const s3 = await getS3Client(creds);
80+
const keysToDelete = new Set();
81+
82+
notesOrIds.forEach(noteOrId => {
83+
const id = typeof noteOrId === 'string' ? noteOrId : noteOrId.id;
84+
const key = getS3ObjectKey(noteOrId.path || noteOrId, creds);
85+
keysToDelete.add(key);
86+
87+
const flatKey = getS3ObjectKey(id, creds);
88+
if (key !== flatKey) {
89+
keysToDelete.add(flatKey);
90+
}
91+
});
92+
93+
if (keysToDelete.size === 0) return;
94+
95+
const params = {
96+
Bucket: creds.bucket,
97+
Delete: {
98+
Objects: Array.from(keysToDelete).map(key => ({ Key: key })),
99+
Quiet: true
100+
}
101+
};
102+
103+
try {
104+
await s3.deleteObjects(params).promise();
105+
console.log(`S3 Bulk: Deleted ${keysToDelete.size} objects from S3 successfully.`);
106+
} catch (err) {
107+
console.error("S3 Bulk Delete Error:", err);
108+
throw new Error(`Failed to perform bulk delete in S3: ${err.code} - ${err.message}`);
109+
}
110+
};
111+
112+
// Export to _GLOBAL
113+
_GLOBAL.deleteNotesFromS3Bulk = deleteNotesFromS3Bulk;
114+
```
115+
116+
### 2.2 Update `deleteNoteFromRemotes` in `src/helpers.js`
117+
Add `skipS3` option to `deleteNoteFromRemotes` to avoid redundant S3 deletion when running S3 bulk delete:
118+
```javascript
119+
async function deleteNoteFromRemotes({noteId, credentials, nostrPrivateKey, nostrRelays, gdriveStore, gitCredentials, remoteMeta, skipS3 = false}) {
120+
// ...
121+
if (credentials?.secretAccessKey && !skipS3) {
122+
promises.push(deleteNoteFromS3(remoteMeta || noteId, credentials));
123+
}
124+
// ...
125+
}
126+
```
127+
128+
### 2.3 Integrate S3 Bulk Delete into `synchronize()` in `src/helpers.js`
129+
In the delete step of `synchronize()`:
130+
1. Check if S3 credentials are configured and there are deleted IDs.
131+
2. If so, call `_GLOBAL.deleteNotesFromS3Bulk(s3NotesOrIds, credentials)` as a separate single promise.
132+
3. Call `deleteNoteFromRemotes` with `skipS3: true` for the standard deletions mapping.
133+
4. Wait for all deletions and the S3 bulk delete to complete using `Promise.allSettled()`.
134+
135+
```javascript
136+
// --- Step 3: Determine which notes to delete from Remotes ---
137+
let s3DeletePromise = Promise.resolve();
138+
const hasS3 = !!credentials?.secretAccessKey;
139+
if (hasS3 && effectiveDeletedNoteIds.length > 0) {
140+
const s3NotesOrIds = effectiveDeletedNoteIds.map(id => remoteMetaMap.get(id) || id);
141+
if (typeof _GLOBAL.deleteNotesFromS3Bulk === 'function') {
142+
s3DeletePromise = _GLOBAL.deleteNotesFromS3Bulk(s3NotesOrIds, credentials);
143+
}
144+
}
145+
146+
const deletePromises = effectiveDeletedNoteIds.map(noteId => {
147+
const remoteMeta = remoteMetaMap.get(noteId);
148+
const gdriveMeta = gdriveMap?.get(noteId);
149+
return deleteNoteFromRemotes({
150+
noteId, credentials, nostrPrivateKey, nostrRelays, gdriveStore, gitCredentials, remoteMeta: gdriveMeta || remoteMeta,
151+
skipS3: hasS3
152+
});
153+
});
154+
155+
// --- Step 4: Execute deletes (deletes are fast, run in parallel) ---
156+
const deleteResults = await Promise.allSettled([...deletePromises, s3DeletePromise]);
157+
```
158+
159+
---
160+
161+
## 3. Backward Compatibility & Verification Strategy
162+
163+
- **Backward Compatibility:** All parameters defaulted (`skipS3 = false`). Legacy calls of `deleteNoteFromRemotes` behave identically.
164+
- **Verification:** Run sync process and check that:
165+
- If a single listing fails, the other services successfully list and we don't trigger upload storms of local notes.
166+
- S3 deletes are bundled and executed via `deleteObjects` efficiently.

‎src/helpers.js‎

Lines changed: 34 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -498,7 +498,7 @@ async function synchronize({notes, deletedNoteIds, isSilent, credentials, nostrP
498498
const { finalIdArray: effectiveDeletedNoteIds, hasChanged: deletedListChanged } = await syncDeletedNoteIds({deletedNoteIds, credentials, gitCredentials, gdriveStore});
499499

500500
// --- Step 1: Get remote state FIRST ---
501-
const { mergedNotes: remoteNoteMetadata, s3Ids, gitIds, nostrIds, gdriveIds, ipfsIds, s3Map, gitMap, nostrMap, gdriveMap, ipfsMap, listingSucceeded } = await listNotes({credentials: ipfsCredentials, nostrPrivateKey, nostrRelays, lastSync, gitCredentials, gdriveStore});
501+
const { mergedNotes: remoteNoteMetadata, s3Ids, gitIds, nostrIds, gdriveIds, ipfsIds, s3Map, gitMap, nostrMap, gdriveMap, ipfsMap, listingSucceeded, failedProviders } = await listNotes({credentials: ipfsCredentials, nostrPrivateKey, nostrRelays, lastSync, gitCredentials, gdriveStore});
502502
const remoteMetaMap = new Map(remoteNoteMetadata.map(m => [m.id, m]));
503503

504504
// --- Step 2: Determine which notes to upload ---
@@ -521,23 +521,23 @@ async function synchronize({notes, deletedNoteIds, isSilent, credentials, nostrP
521521
// if localDate > remoteDate, it's newer than ALL remotes.
522522
// if localDate == remoteDate, it might still be newer than SOME remotes if they are inconsistent).
523523

524-
if (gitCredentials?.repoUrl) {
524+
if (gitCredentials?.repoUrl && !failedProviders.git) {
525525
const gitMeta = gitMap.get(localNote.id);
526526
if (!gitMeta || localDate > new Date(gitMeta.updatedAt || gitMeta.createdAt)) return true;
527527
}
528-
if (credentials.secretAccessKey) {
528+
if (credentials.secretAccessKey && !failedProviders.s3) {
529529
const s3Meta = s3Map.get(localNote.id);
530530
if (!s3Meta || localDate > new Date(s3Meta.updatedAt)) return true;
531531
}
532-
if (nostrPrivateKey) {
532+
if (nostrPrivateKey && !failedProviders.nostr) {
533533
const nostrMeta = nostrMap.get(localNote.id);
534534
if (!nostrMeta || localDate > new Date(nostrMeta.updatedAt || nostrMeta.createdAt)) return true;
535535
}
536-
if (gdriveStore?.connected) {
536+
if (gdriveStore?.connected && !failedProviders.gdrive) {
537537
const gdriveMeta = gdriveMap.get(localNote.id);
538538
if (!gdriveMeta || localDate > new Date(gdriveMeta.updatedAt || gdriveMeta.createdAt)) return true;
539539
}
540-
if (credentials.pinataJwt || credentials.pinataApiKey) {
540+
if ((credentials.pinataJwt || credentials.pinataApiKey) && !failedProviders.ipfs) {
541541
const ipfsMeta = ipfsMap.get(localNote.id);
542542
if (!ipfsMeta || localDate > new Date(ipfsMeta.updatedAt)) return true;
543543
}
@@ -591,19 +591,28 @@ async function synchronize({notes, deletedNoteIds, isSilent, credentials, nostrP
591591
}
592592

593593
// --- Step 3: Determine which notes to delete from Remotes ---
594+
let s3DeletePromise = Promise.resolve();
595+
const hasS3 = !!credentials?.secretAccessKey;
596+
if (hasS3 && effectiveDeletedNoteIds.length > 0) {
597+
const s3NotesOrIds = effectiveDeletedNoteIds.map(id => remoteMetaMap.get(id) || id);
598+
if (typeof _GLOBAL.deleteNotesFromS3Bulk === 'function') {
599+
s3DeletePromise = _GLOBAL.deleteNotesFromS3Bulk(s3NotesOrIds, credentials);
600+
}
601+
}
602+
594603
const deletePromises = effectiveDeletedNoteIds.map(noteId => {
595604
const remoteMeta = remoteMetaMap.get(noteId);
596605
const gdriveMeta = gdriveMap?.get(noteId);
597606
return deleteNoteFromRemotes({
598-
noteId, credentials, nostrPrivateKey, nostrRelays, gdriveStore, gitCredentials, remoteMeta: gdriveMeta || remoteMeta
607+
noteId, credentials, nostrPrivateKey, nostrRelays, gdriveStore, gitCredentials, remoteMeta: gdriveMeta || remoteMeta,
608+
skipS3: hasS3
599609
});
600610
});
601611

602612
// --- Step 4: Execute deletes (deletes are fast, run in parallel) ---
603-
const deleteResults = await Promise.allSettled(deletePromises);
613+
const deleteResults = await Promise.allSettled([...deletePromises, s3DeletePromise]);
604614

605-
// Combine results for counting
606-
const uploadAndDeleteResults = [...uploadResults, ...deleteResults];
615+
const s3BulkDeleteSucceeded = !hasS3 || (deleteResults[deleteResults.length - 1]?.status === 'fulfilled');
607616

608617
let successfulUploadedCount = 0;
609618
const successfulDeletedIds = [];
@@ -616,10 +625,10 @@ async function synchronize({notes, deletedNoteIds, isSilent, credentials, nostrP
616625
}
617626

618627
// Count successful deletes (remaining items are deletes)
619-
for (let i = uploadResults.length; i < uploadAndDeleteResults.length; i++) {
620-
if (uploadAndDeleteResults[i].status === 'fulfilled') {
621-
const deletedIndex = i - uploadResults.length;
622-
successfulDeletedIds.push(effectiveDeletedNoteIds[deletedIndex]);
628+
for (let i = 0; i < deletePromises.length; i++) {
629+
const deleteResult = deleteResults[i];
630+
if (deleteResult.status === 'fulfilled' && s3BulkDeleteSucceeded) {
631+
successfulDeletedIds.push(effectiveDeletedNoteIds[i]);
623632
}
624633
}
625634

@@ -1015,6 +1024,13 @@ async function listNotes({credentials, nostrPrivateKey, nostrRelays, lastSync, g
10151024
});
10161025

10171026
const listingSucceeded = [s3Res, nostrRes, gitRes, gdriveRes, ipfsRes].every(r => r.status === 'fulfilled');
1027+
const failedProviders = {
1028+
s3: s3Res.status === 'rejected',
1029+
nostr: nostrRes.status === 'rejected',
1030+
git: gitRes.status === 'rejected',
1031+
gdrive: gdriveRes.status === 'rejected',
1032+
ipfs: ipfsRes.status === 'rejected'
1033+
};
10181034

10191035

10201036
return {
@@ -1029,7 +1045,8 @@ async function listNotes({credentials, nostrPrivateKey, nostrRelays, lastSync, g
10291045
nostrMap: new Map(nostrNotes.map(n => [n.id, n])),
10301046
gdriveMap: new Map(gdriveNotes.map(n => [n.id, n])),
10311047
ipfsMap: new Map(ipfsNotes.map(n => [n.id, n])),
1032-
listingSucceeded
1048+
listingSucceeded,
1049+
failedProviders
10331050
};
10341051
}
10351052

@@ -1075,15 +1092,15 @@ async function listImages({credentials, nostrPrivateKey, nostrRelays}) {
10751092
return Array.from(mergedImages.values());
10761093
}
10771094

1078-
async function deleteNoteFromRemotes({noteId, credentials, nostrPrivateKey, nostrRelays, gdriveStore, gitCredentials, remoteMeta}) {
1095+
async function deleteNoteFromRemotes({noteId, credentials, nostrPrivateKey, nostrRelays, gdriveStore, gitCredentials, remoteMeta, skipS3 = false}) {
10791096
const promises = [];
10801097

10811098
// Remote deletions
10821099
if (gdriveStore.connected && typeof _GLOBAL.deleteNoteFromGoogleDrive === 'function') {
10831100
promises.push(_GLOBAL.deleteNoteFromGoogleDrive(noteId, remoteMeta));
10841101
}
10851102

1086-
if (credentials?.secretAccessKey) {
1103+
if (credentials?.secretAccessKey && !skipS3) {
10871104
promises.push(deleteNoteFromS3(remoteMeta || noteId, credentials));
10881105
}
10891106

‎src/index.html‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
<link rel="dns-prefetch" href="https://cdn.jsdelivr.net">
3535
<link rel="preconnect" href="https://maxcdn.bootstrapcdn.com" crossorigin>
3636
</head>
37-
<body id="main-app" class="flex flex-col min-h-screen" :class="{'dark': darkMode, '': !darkMode}" x-data="mainApp()" x-init="init()">
37+
<body id="main-app" class="flex flex-col min-h-screen font-mono" :class="{'dark': darkMode, '': !darkMode}" x-data="mainApp()" x-init="init()">
3838
<!-- Biometric Lock Overlay -->
3939
<div x-show="isLocked"
4040
x-transition:enter="transition ease-out duration-300"

‎src/s3.js‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,42 @@ const deleteNoteFromS3 = async (noteOrId, creds) => {
210210
}
211211
};
212212

213+
const deleteNotesFromS3Bulk = async (notesOrIds, creds) => {
214+
if (!creds?.secretAccessKey || !notesOrIds || notesOrIds.length === 0) return;
215+
216+
const s3 = await getS3Client(creds);
217+
const keysToDelete = new Set();
218+
219+
notesOrIds.forEach(noteOrId => {
220+
const id = typeof noteOrId === 'string' ? noteOrId : noteOrId.id;
221+
const key = getS3ObjectKey(noteOrId.path || noteOrId, creds);
222+
keysToDelete.add(key);
223+
224+
const flatKey = getS3ObjectKey(id, creds);
225+
if (key !== flatKey) {
226+
keysToDelete.add(flatKey);
227+
}
228+
});
229+
230+
if (keysToDelete.size === 0) return;
231+
232+
const params = {
233+
Bucket: creds.bucket,
234+
Delete: {
235+
Objects: Array.from(keysToDelete).map(key => ({ Key: key })),
236+
Quiet: true
237+
}
238+
};
239+
240+
try {
241+
await s3.deleteObjects(params).promise();
242+
console.log(`S3 Bulk: Deleted ${keysToDelete.size} objects from S3 successfully.`);
243+
} catch (err) {
244+
console.error("S3 Bulk Delete Error:", err);
245+
throw new Error(`Failed to perform bulk delete in S3: ${err.code} - ${err.message}`);
246+
}
247+
};
248+
213249
// --- S3 Functions for Images ---
214250
const getImageS3ObjectKey = (imageId, imageType, creds) => {
215251
const path = creds.subfolder ? `${creds.subfolder.replace(/\/$/, '')}/` : '';
@@ -413,6 +449,7 @@ _GLOBAL.listNotesInS3 = listNotesInS3;
413449
_GLOBAL.downloadNoteFromS3 = downloadNoteFromS3;
414450
_GLOBAL.getNoteMetadataFromS3 = getNoteMetadataFromS3;
415451
_GLOBAL.deleteNoteFromS3 = deleteNoteFromS3;
452+
_GLOBAL.deleteNotesFromS3Bulk = deleteNotesFromS3Bulk;
416453
_GLOBAL.uploadImageToS3 = uploadImageToS3;
417454
_GLOBAL.downloadImageFromS3 = downloadImageFromS3;
418455
_GLOBAL.listImagesInS3 = listImagesInS3;

0 commit comments

Comments
 (0)