Manually set last seen timestamp as sync integrates

- This was prev being done within sync itself, though the problem there was that the timestamp was set before the updates were integrated in the local DB. Which meant that it could get into states where sync got interrupted between when the timestamp was written and when the updates were integrated, thus skipping those updates the next time sync started

Author: poltak

src/background-script/setup.ts

@@ -455,11 +455,12 @@ export function createBackgroundModules(options: {
         backend:
             options.personalCloudBackend ??
             new FirestorePersonalCloudBackend({
+                getFirebase,
+                getServerStorageManager,
                 personalCloudService: firebaseService<PersonalCloudService>(
                     'personalCloud',
                     callFirebaseFunction,
                 ),
-                getServerStorageManager,
                 getCurrentSchemaVersion: () =>
                     getCurrentSchemaVersion(options.storageManager),
                 userChanges: () => authChanges(auth.authService),
@@ -475,10 +476,8 @@ export function createBackgroundModules(options: {
                         .doc(currentUser.id)
                         .collection('objects')
                 },
-                getLastUpdateSeenTime: () =>
+                getLastUpdateProcessedTime: () =>
                     personalCloudSettingStore.get('lastSeen'),
-                setLastUpdateSeenTime: (lastSeen) =>
-                    personalCloudSettingStore.set('lastSeen', lastSeen),
                 getDeviceId: async () => personalCloud.deviceId!,
                 getClientDeviceType: () => PersonalDeviceType.DesktopBrowser,
             }),

src/personal-cloud/background/index.test.ts

@@ -1,5 +1,5 @@
 import fs from 'fs'
-import { setupSyncBackgroundTest } from './index.tests'
+import { setupSyncBackgroundTest, BASE_TIMESTAMP } from './index.tests'
 import { StorexPersonalCloudBackend } from '@worldbrain/memex-common/lib/personal-cloud/backend/storex'
 import { TEST_USER } from '@worldbrain/memex-common/lib/authentication/dev'
 import { BackgroundIntegrationTestSetup } from 'src/tests/integration-tests'
@@ -16,11 +16,6 @@ import {
     TEST_PDF_PAGE_TEXTS,
 } from 'src/tests/test.data'
 import { blobToJson } from 'src/util/blob-utils'
-import { Annotation } from '../../annotations/types'
-import {
-    PersonalCloudErrorType,
-    DataChangeType,
-} from '../../../external/@worldbrain/memex-common/ts/personal-cloud/storage/types'
 
 describe('Personal cloud', () => {
     const testFullPage = async (testOptions: {
@@ -411,14 +406,35 @@ describe('Personal cloud', () => {
             id: 1,
             name: 'list 1',
         })
+
+        expect(
+            await setups[1].backgroundModules.personalCloud.options.settingStore.get(
+                'lastSeen',
+            ),
+        ).toEqual(null)
+
         await waitForSync()
+
+        expect(
+            await setups[1].backgroundModules.personalCloud.options.settingStore.get(
+                'lastSeen',
+            ),
+        ).toEqual(BASE_TIMESTAMP)
+
         await assertCacheEntries([1])
 
         await setups[0].backgroundModules.customLists.createCustomList({
             id: 2,
             name: 'list 2',
         })
         await waitForSync()
+
+        expect(
+            await setups[1].backgroundModules.personalCloud.options.settingStore.get(
+                'lastSeen',
+            ),
+        ).toEqual(BASE_TIMESTAMP + 1)
+
         await assertCacheEntries([2, 1])
 
         await setups[0].backgroundModules.customLists.updateList({
@@ -427,13 +443,27 @@ describe('Personal cloud', () => {
             newName: 'list 1 updated',
         })
         await waitForSync()
+
+        expect(
+            await setups[1].backgroundModules.personalCloud.options.settingStore.get(
+                'lastSeen',
+            ),
+        ).toEqual(BASE_TIMESTAMP + 2)
+
         await assertCacheEntries([2, 1]) // Updates should not affect cache
 
         await setups[0].backgroundModules.customLists.createCustomList({
             id: 3,
             name: 'list 3',
         })
         await waitForSync()
+
+        expect(
+            await setups[1].backgroundModules.personalCloud.options.settingStore.get(
+                'lastSeen',
+            ),
+        ).toEqual(BASE_TIMESTAMP + 3)
+
         await assertCacheEntries([3, 2, 1])
     })
 })

src/personal-cloud/background/index.tests.ts

@@ -21,6 +21,8 @@ import { STORAGE_VERSIONS } from 'src/storage/constants'
 import { createServices } from 'src/services'
 import { PersonalDeviceType } from '@worldbrain/memex-common/lib/personal-cloud/storage/types'
 
+export const BASE_TIMESTAMP = 555
+
 const debug = (...args: any[]) => console['log'](...args, '\n\n\n')
 
 type SyncTestSequence = SyncTestStep[]
@@ -282,7 +284,7 @@ export async function setupSyncBackgroundTest(
     const serverStorage = await getServerStorage()
     const cloudHub = new PersonalCloudHub()
 
-    let now = 555
+    let now = BASE_TIMESTAMP
     const getNow = () => now++
     const setups: BackgroundIntegrationTestSetup[] = []
     for (let i = 0; i < options.deviceCount; ++i) {

src/personal-cloud/background/index.ts

@@ -278,15 +278,19 @@ export class PersonalCloudBackground {
     }
 
     async integrateAllUpdates(): Promise<void> {
-        const updateBatch = await this.options.backend.bulkDownloadUpdates()
-        return this.integrateUpdates(updateBatch)
+        const { backend, settingStore } = this.options
+        const { batch, lastSeen } = await backend.bulkDownloadUpdates()
+        await this.integrateUpdates(batch)
+        await settingStore.set('lastSeen', lastSeen)
     }
 
     async integrateContinuously() {
+        const { backend, settingStore } = this.options
         try {
-            for await (const updates of this.options.backend.streamUpdates()) {
+            for await (const { batch, lastSeen } of backend.streamUpdates()) {
                 try {
-                    await this.integrateUpdates(updates)
+                    await this.integrateUpdates(batch)
+                    await settingStore.set('lastSeen', lastSeen)
                 } catch (err) {
                     if (this.strictErrorReporting) {
                         this._integrationError = err