keep performance merge logic on native

Author: chrispader

lib/Onyx.ts

@@ -1,4 +1,3 @@
-/* eslint-disable no-continue */
 import _ from 'underscore';
 import lodashPick from 'lodash/pick';
 import * as Logger from './Logger';
@@ -35,6 +34,7 @@ import type {Connection} from './OnyxConnectionManager';
 import connectionManager from './OnyxConnectionManager';
 import * as GlobalSettings from './GlobalSettings';
 import decorateWithMetrics from './metrics';
+import OnyxMerge from './OnyxMerge.native';
 
 /** Initialize the store with actions and listening for storage events */
 function init({
@@ -325,26 +325,7 @@ function merge<TKey extends OnyxKey>(key: TKey, changes: OnyxMergeInput<TKey>):
                 return Promise.resolve();
             }
 
-            const {result: mergedValue} = OnyxUtils.mergeChanges(validChanges, existingValue);
-
-            // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge.
-            const hasChanged = cache.hasValueChanged(key, mergedValue);
-
-            // Logging properties only since values could be sensitive things we don't want to log.
-            Logger.logInfo(`merge called for key: ${key}${_.isObject(mergedValue) ? ` properties: ${_.keys(mergedValue).join(',')}` : ''} hasChanged: ${hasChanged}`);
-
-            // This approach prioritizes fast UI changes without waiting for data to be stored in device storage.
-            const updatePromise = OnyxUtils.broadcastUpdate(key, mergedValue as OnyxValue<TKey>, hasChanged);
-
-            // If the value has not changed, calling Storage.setItem() would be redundant and a waste of performance, so return early instead.
-            if (!hasChanged) {
-                return updatePromise;
-            }
-
-            return Storage.setItem(key, mergedValue as OnyxValue<TKey>).then(() => {
-                OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE, key, changes, mergedValue);
-                return updatePromise;
-            });
+            return OnyxMerge.applyMerge(key, existingValue, validChanges);
         } catch (error) {
             Logger.logAlert(`An error occurred while applying merge for key: ${key}, Error: ${error}`);
             return Promise.resolve();

lib/OnyxMerge.native.ts

@@ -0,0 +1,43 @@
+import _ from 'underscore';
+import * as Logger from './Logger';
+import OnyxUtils from './OnyxUtils';
+import type {OnyxKey, OnyxValue} from './types';
+import cache from './OnyxCache';
+import Storage from './storage';
+
+function applyMerge<TKey extends OnyxKey>(key: TKey, existingValue: OnyxValue<TKey>, validChanges: unknown[]): Promise<void> {
+    // If any of the changes is null, we need to discard the existing value.
+    const baseValue = validChanges.includes(null) ? undefined : existingValue;
+
+    // We first batch the changes into a single change with object removal marks,
+    // so that SQLite can merge the changes more efficiently.
+    const {result: batchedChanges, replaceNullPatches} = OnyxUtils.mergeAndMarkChanges(validChanges);
+
+    // We then merge the batched changes with the existing value, because we need to final merged value to broadcast to subscribers.
+    const {result: mergedValue} = OnyxUtils.mergeChanges([batchedChanges], baseValue);
+
+    // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge.
+    const hasChanged = cache.hasValueChanged(key, mergedValue);
+
+    // Logging properties only since values could be sensitive things we don't want to log.
+    Logger.logInfo(`merge called for key: ${key}${_.isObject(mergedValue) ? ` properties: ${_.keys(mergedValue).join(',')}` : ''} hasChanged: ${hasChanged}`);
+
+    // This approach prioritizes fast UI changes without waiting for data to be stored in device storage.
+    const updatePromise = OnyxUtils.broadcastUpdate(key, mergedValue as OnyxValue<TKey>, hasChanged);
+
+    // If the value has not changed, calling Storage.setItem() would be redundant and a waste of performance, so return early instead.
+    if (!hasChanged) {
+        return updatePromise;
+    }
+
+    return Storage.mergeItem(key, batchedChanges as OnyxValue<TKey>, replaceNullPatches).then(() => {
+        OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE, key, validChanges, mergedValue);
+        return updatePromise;
+    });
+}
+
+const OnyxMerge = {
+    applyMerge,
+};
+
+export default OnyxMerge;

lib/OnyxMerge.ts

@@ -0,0 +1,35 @@
+import _ from 'underscore';
+import * as Logger from './Logger';
+import OnyxUtils from './OnyxUtils';
+import type {OnyxKey, OnyxValue} from './types';
+import cache from './OnyxCache';
+import Storage from './storage';
+
+function applyMerge<TKey extends OnyxKey>(key: TKey, existingValue: OnyxValue<TKey>, validChanges: unknown[]): Promise<void> {
+    const {result: mergedValue} = OnyxUtils.mergeChanges(validChanges, existingValue);
+
+    // In cache, we don't want to remove the key if it's null to improve performance and speed up the next merge.
+    const hasChanged = cache.hasValueChanged(key, mergedValue);
+
+    // Logging properties only since values could be sensitive things we don't want to log.
+    Logger.logInfo(`merge called for key: ${key}${_.isObject(mergedValue) ? ` properties: ${_.keys(mergedValue).join(',')}` : ''} hasChanged: ${hasChanged}`);
+
+    // This approach prioritizes fast UI changes without waiting for data to be stored in device storage.
+    const updatePromise = OnyxUtils.broadcastUpdate(key, mergedValue as OnyxValue<TKey>, hasChanged);
+
+    // If the value has not changed, calling Storage.setItem() would be redundant and a waste of performance, so return early instead.
+    if (!hasChanged) {
+        return updatePromise;
+    }
+
+    return Storage.setItem(key, mergedValue as OnyxValue<TKey>).then(() => {
+        OnyxUtils.sendActionToDevTools(OnyxUtils.METHOD.MERGE, key, validChanges, mergedValue);
+        return updatePromise;
+    });
+}
+
+const OnyxMerge = {
+    applyMerge,
+};
+
+export default OnyxMerge;

lib/OnyxUtils.ts

@@ -1236,14 +1236,11 @@ function prepareKeyValuePairsForStorage(
     return pairs;
 }
 
-function mergeChanges<TValue extends OnyxInput<OnyxKey> | undefined, TChange extends OnyxInput<OnyxKey> | undefined>(changes: TChange[], existingValue?: TValue): FastMergeResult<TChange> {
+function mergeChanges<TValue extends OnyxInput<OnyxKey> | undefined>(changes: TValue[], existingValue?: TValue): FastMergeResult<TValue> {
     return applyMerge('merge', changes, existingValue);
 }
 
-function mergeAndMarkChanges<TValue extends OnyxInput<OnyxKey> | undefined, TChange extends OnyxInput<OnyxKey> | undefined>(
-    changes: TChange[],
-    existingValue?: TValue,
-): FastMergeResult<TChange> {
+function mergeAndMarkChanges<TValue extends OnyxInput<OnyxKey> | undefined>(changes: TValue[], existingValue?: TValue): FastMergeResult<TValue> {
     return applyMerge('mark', changes, existingValue);
 }
 
@@ -1253,11 +1250,7 @@ function mergeAndMarkChanges<TValue extends OnyxInput<OnyxKey> | undefined, TCha
  * @param changes Array of changes that should be merged
  * @param existingValue The existing value that should be merged with the changes
  */
-function applyMerge<TValue extends OnyxInput<OnyxKey> | undefined, TChange extends OnyxInput<OnyxKey> | undefined>(
-    mode: 'merge' | 'mark',
-    changes: TChange[],
-    existingValue?: TValue,
-): FastMergeResult<TChange> {
+function applyMerge<TValue extends OnyxInput<OnyxKey> | undefined>(mode: 'merge' | 'mark', changes: TValue[], existingValue?: TValue): FastMergeResult<TValue> {
     const lastChange = changes?.at(-1);
 
     if (Array.isArray(lastChange)) {
@@ -1266,7 +1259,7 @@ function applyMerge<TValue extends OnyxInput<OnyxKey> | undefined, TChange exten
 
     if (changes.some((change) => change && typeof change === 'object')) {
         // Object values are then merged one after the other
-        return changes.reduce<FastMergeResult<TChange>>(
+        return changes.reduce<FastMergeResult<TValue>>(
             (modifiedData, change) => {
                 const options: FastMergeOptions = mode === 'merge' ? {shouldRemoveNestedNulls: true, objectRemovalMode: 'replace'} : {objectRemovalMode: 'mark'};
                 const {result, replaceNullPatches} = utils.fastMerge(modifiedData.result, change, options);
@@ -1279,15 +1272,15 @@ function applyMerge<TValue extends OnyxInput<OnyxKey> | undefined, TChange exten
                 return modifiedData;
             },
             {
-                result: (existingValue ?? {}) as TChange,
+                result: (existingValue ?? {}) as TValue,
                 replaceNullPatches: [],
             },
         );
     }
 
     // If we have anything else we can't merge it so we'll
     // simply return the last value that was queued
-    return {result: lastChange as TChange, replaceNullPatches: []};
+    return {result: lastChange as TValue, replaceNullPatches: []};
 }
 
 /**