[#87] Implement better syncing for events and blobs

Changelog: enhancement

Author: brandon-futo

.gitlab/ci/build_js_sdks.yml

@@ -30,6 +30,13 @@ js-sdks-format-check:
     - pnpm format
   allow_failure: true
 
+js-core-tests-check:
+  extends: .js-sdks-base
+  stage: check
+  allow_failure: true
+  script:
+    - pnpm --filter js-core test:run
+
 js-sdks-build:
   extends: .js-sdks-base
   stage: build

apps/polycentric/src/common/lib/polycentric-hooks/PolycentricProvider.tsx

@@ -180,8 +180,6 @@ export function PolycentricProvider({
 
         if (cancelled) return;
 
-        c.servers = [DEFAULT_SERVER];
-
         const s = createPolycentricStore(c);
         await s.getState().refreshIdentities();
 

apps/polycentric/src/features/moderation/hooks/useReportAction.ts

@@ -1,5 +1,5 @@
 import { hexToBytes, usePolycentric } from '@/src/common/lib/polycentric-hooks';
-import { COLLECTION, v2 } from '@polycentric/react-native';
+import { COLLECTION, v2, SyncStrategy } from '@polycentric/react-native';
 import { useState } from 'react';
 
 export default function useReportAction() {
@@ -36,7 +36,7 @@ export default function useReportAction() {
       });
 
       // Persist the content first: the event references it by digest, and
-      // `push()` looks the content up from the local store to attach it.
+      // `sync()` looks the content up from the local store to attach it.
       await client.contentManager.save(content);
 
       const event = await client.buildEvent(content, COLLECTION.REPORTS);
@@ -48,7 +48,7 @@ export default function useReportAction() {
       // Delivery to servers is best-effort — the report is already saved
       // locally and will be pushed on the next sync if this fails.
       try {
-        await client.push();
+        await client.sync(SyncStrategy.PARTIAL_PUSH);
       } catch (e) {
         console.warn('Failed to push report to servers:', e);
       } finally {

apps/polycentric/src/features/post/hooks/usePostActions.ts

@@ -4,7 +4,7 @@ import {
   usePolycentric,
 } from '@/src/common/lib/polycentric-hooks';
 import { invalidateQuery } from '@/src/common/query/hooks/useQuery';
-import { COLLECTION, v2 } from '@polycentric/react-native';
+import { COLLECTION, v2, SyncStrategy } from '@polycentric/react-native';
 import { router, useSegments } from 'expo-router';
 import { feedQueryKeys } from '../../feed/hooks/feedCache';
 
@@ -44,7 +44,7 @@ export default function usePostActions(post: PostData): PostActions {
     await client.commitEvent(signedDelete, deleteContent);
 
     // TODO: do we care if push failed?
-    await client.push();
+    await client.sync(SyncStrategy.PARTIAL_PUSH);
   };
 
   const invalidateFeeds = (identity: string) => {

packages/js-core/package.json

@@ -16,7 +16,8 @@
     "generate": "mkdir -p ./src/proto && protoc --ts_out=./src/proto --proto_path=../../protos --experimental_allow_proto3_optional ../../protos/polycentric.proto ../../protos/polycentric/v2/keypair.proto ../../protos/polycentric/v2/event_key.proto ../../protos/polycentric/v2/identity.proto ../../protos/polycentric/v2/content.proto ../../protos/polycentric/v2/events.proto ../../protos/polycentric/v2/feeds.proto ../../protos/polycentric/v2/server.proto ../../protos/polycentric/v2/pairing_service.proto ../../protos/polycentric/v2/notifications.proto",
     "build": "tsc -b && webpack",
     "dev": "tsc -b && webpack --watch",
-    "test": "echo 'No tests'"
+    "test": "vitest",
+    "test:run": "vitest run"
   },
   "dependencies": {
     "@noble/curves": "catalog:",
@@ -34,6 +35,7 @@
     "@protobuf-ts/plugin": "^2.11.1",
     "ts-loader": "^9.5.4",
     "typescript": "catalog:",
+    "vitest": "^3.2.6",
     "webpack": "^5.100.1",
     "webpack-cli": "^7.0.0"
   }

packages/js-core/src/__mocks__/rs-core-uniffi-web-generated.ts

@@ -0,0 +1,14 @@
+export enum QueryStatus {
+  Loading = 1,
+  Success = 2,
+  Error = 3,
+}
+
+class ListEvents {
+  readonly inner: [unknown];
+  constructor(args: unknown) {
+    this.inner = [args];
+  }
+}
+
+export const Query = { ListEvents };

packages/js-core/src/client-internal/content-manager.ts

@@ -33,15 +33,16 @@ export class ContentManager {
   }
 
   /**
-   * Download any blobs that we don't have locally from this content.
+   * Download any blobs in the array that we don't have locally.
    * This is used so that blobs of an identity will eventually
    * persist on other devices in that identity.
    */
-  async pullBlobs(content: Proto.Content): Promise<void> {
-    const digests = this.collectBlobDigests(content);
+  async pullBlobs(blobs: Proto.Blob[]): Promise<void> {
+    const digests = blobs.map((b) => b.digest).filter((d) => !!d);
+
     if (digests.length === 0) return;
 
-    await Promise.all(
+    await Promise.allSettled(
       digests.map(async (digest) => {
         try {
           if (await this.client.filestoreDriver.has(digest)) return;
@@ -56,14 +57,14 @@ export class ContentManager {
   }
 
   /**
-   * Collect all blob digests referenced in a post or profile update
+   * Collect all blobs referenced in a post or profile update
    */
-  private collectBlobDigests(content: Proto.Content): Proto.ContentDigest[] {
-    const out: Proto.ContentDigest[] = [];
+  static collectBlobs(content: Proto.Content): Proto.Blob[] {
+    const out: Proto.Blob[] = [];
     const pushSet = (set?: Proto.ImageSet) => {
       if (!set) return;
       for (const img of set.images) {
-        if (img.blob?.digest) out.push(img.blob.digest);
+        if (img.blob) out.push(img.blob);
       }
     };
     const body = content.contentBody;

packages/js-core/src/client-internal/identity-manager.ts

@@ -1,6 +1,6 @@
 import { sha256 } from '@noble/hashes/sha2.js';
 import { Query, QueryStatus } from '@polycentric/rs-core-uniffi-web/generated';
-import { COLLECTION } from '../constants';
+import { COLLECTION, SyncStrategy } from '../constants';
 import type { PolycentricClient } from '../polycentric-client';
 import * as Proto from '../proto/v2';
 import { bytesEqual } from '../utils/bytes';
@@ -137,7 +137,7 @@ export class IdentityManager {
 
     const signedEvent = await this.client.signEvent(event);
     await this.client.commitEvent(signedEvent, content);
-    await this.client.push();
+    await this.client.sync(SyncStrategy.PARTIAL_PUSH);
 
     return { identityKey: resolvedIdentityKey, signedEvent };
   }
@@ -252,7 +252,7 @@ export class IdentityManager {
     }
 
     await this.client.setActiveIdentityKey(identityKey);
-    await this.client.pull();
+    await this.client.sync(SyncStrategy.PARTIAL_PULL);
 
     // Re-publish the same identity document signed by our own key,
     // proving this key acknowledged its membership.

packages/js-core/src/constants.ts

@@ -23,6 +23,31 @@ export enum HydrationStrategy {
   LAZY = 'lazy',
 }
 
+/**
+ * Specifies how to sync events and blobs for an identity between the local store
+ * and the remote servers.
+ */
+export enum SyncStrategy {
+  /** Push and pull all events. */
+  FULL,
+  /** Push all events. */
+  FULL_PUSH,
+  /** Pull all events. */
+  FULL_PULL,
+  /** Push and pull only events believed to be missing. */
+  PARTIAL,
+  /**
+   * Push only events believed to be missing.
+   * No events are pulled.
+   */
+  PARTIAL_PUSH,
+  /**
+   * Pull only events believed to be missing.
+   * No events are pushed.
+   */
+  PARTIAL_PULL,
+}
+
 export const Defaults = {
   DB_NAME: 'polycentric-database',
   HYDRATION: {

packages/js-core/src/index.ts

@@ -43,7 +43,12 @@ export * from './errors';
 export * as Errors from './errors';
 export * from './utils';
 
-export { KEY_TYPE, COLLECTION, HydrationStrategy } from './constants';
+export {
+  KEY_TYPE,
+  COLLECTION,
+  HydrationStrategy,
+  SyncStrategy,
+} from './constants';
 
 export {
   ClientState,