refactor: Optionally support murmur3 + implement capabilities negotiation

Signed-off-by: Marcel Klehr <mklehr@gmx.net>

Author: marcelklehr

src/lib/CachingTreeWrapper.ts

@@ -1,4 +1,4 @@
-import { CachingResource, OrderFolderResource } from './interfaces/Resource'
+import { CachingResource, ICapabilities, IHashSettings, OrderFolderResource } from './interfaces/Resource'
 import { Bookmark, Folder, ItemLocation } from './Tree'
 import CacheTree from './CacheTree'
 import Ordering from './interfaces/Ordering'
@@ -74,4 +74,12 @@ export default class CachingTreeWrapper implements OrderFolderResource<typeof It
   getCacheTree(): Promise<Folder<typeof ItemLocation.LOCAL>> {
     return this.cacheTree.getBookmarksTree()
   }
+
+  getCapabilities(): Promise<ICapabilities> {
+    return this.innerTree.getCapabilities()
+  }
+
+  setHashSettings(hashSettings: IHashSettings): void {
+    this.innerTree.setHashSettings(hashSettings)
+  }
 }

src/lib/Crypto.ts

@@ -1,9 +1,17 @@
 import { fromUint8Array, toUint8Array } from 'js-base64'
+import { murmurhash3_32_gc } from './murmurhash3'
 
 export default class Crypto {
   static iterations = 250000
   static ivLength = 16
 
+  static async murmurHash3(message: string): Promise<string> {
+    const buf32 = new Uint32Array([murmurhash3_32_gc(message, 0)])
+    const buf8 = new Uint8Array(buf32.buffer)
+    buf8.reverse()
+    return this.bufferToHexstr(buf8)
+  }
+
   static async sha256(message: string): Promise<string> {
     const msgBuffer = new TextEncoder().encode(message) // encode as UTF-8
     const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer) // hash the message

src/lib/Scanner.ts

@@ -2,6 +2,7 @@ import * as Parallel from 'async-parallel'
 import Diff, { ActionType, CreateAction, MoveAction, RemoveAction, ReorderAction, UpdateAction } from './Diff'
 import { Bookmark, Folder, ItemLocation, ItemType, TItem, TItemLocation } from './Tree'
 import Logger from './Logger'
+import { IHashSettings } from './interfaces/Resource'
 
 export interface ScanResult<L1 extends TItemLocation, L2 extends TItemLocation> {
   CREATE: Diff<L1, L2, CreateAction<L1, L2>>
@@ -15,17 +16,17 @@ export default class Scanner<L1 extends TItemLocation, L2 extends TItemLocation>
   private oldTree: TItem<L1>
   private newTree: TItem<L2>
   private mergeable: (i1: TItem<TItemLocation>, i2: TItem<TItemLocation>) => boolean
-  private preserveOrder: boolean
+  private hashSettings: IHashSettings
   private checkHashes: boolean
   private hasCache: boolean
 
   private result: ScanResult<L2, L1>
 
-  constructor(oldTree:TItem<L1>, newTree:TItem<L2>, mergeable:(i1:TItem<TItemLocation>, i2:TItem<TItemLocation>)=>boolean, preserveOrder:boolean, checkHashes = true, hasCache = true) {
+  constructor(oldTree:TItem<L1>, newTree:TItem<L2>, mergeable:(i1:TItem<TItemLocation>, i2:TItem<TItemLocation>)=>boolean, hashSettings: IHashSettings, checkHashes = true, hasCache = true) {
     this.oldTree = oldTree
     this.newTree = newTree
     this.mergeable = mergeable
-    this.preserveOrder = preserveOrder
+    this.hashSettings = hashSettings
     this.checkHashes = typeof checkHashes === 'undefined' ? true : checkHashes
     this.hasCache = hasCache
     this.result = {
@@ -136,14 +137,14 @@ export default class Scanner<L1 extends TItemLocation, L2 extends TItemLocation>
   }
 
   async bookmarkHasChanged(oldBookmark:Bookmark<L1>, newBookmark:Bookmark<L2>):Promise<boolean> {
-    const oldHash = await oldBookmark.hash()
-    const newHash = await newBookmark.hash()
+    const oldHash = await oldBookmark.hash(this.hashSettings)
+    const newHash = await newBookmark.hash(this.hashSettings)
     return oldHash !== newHash
   }
 
   async folderHasChanged(oldFolder:Folder<L1>, newFolder:Folder<L2>):Promise<boolean> {
-    const oldHash = await oldFolder.hash(this.preserveOrder)
-    const newHash = await newFolder.hash(this.preserveOrder)
+    const oldHash = await oldFolder.hash(this.hashSettings)
+    const newHash = await newFolder.hash(this.hashSettings)
     return oldHash !== newHash
   }
 

src/lib/Tree.ts

@@ -1,6 +1,6 @@
 import Crypto from './Crypto'
 import Logger from './Logger'
-import TResource from './interfaces/Resource'
+import TResource, { IHashSettings } from './interfaces/Resource'
 import * as Parallel from 'async-parallel'
 
 const STRANGE_PROTOCOLS = ['data:', 'javascript:', 'about:', 'chrome:', 'file:']
@@ -37,7 +37,7 @@ export class Bookmark<L extends TItemLocation> {
   public tags: string[]
   public location: L
   public isRoot = false
-  private hashValue: string
+  private hashValue: Record<string, string>
 
   constructor({ id, parentId, url, title, tags, location }: { id:string|number, parentId:string|number, url:string, title:string, tags?: string[], location: L }) {
     this.id = id
@@ -78,13 +78,19 @@ export class Bookmark<L extends TItemLocation> {
     return 0
   }
 
-  async hash():Promise<string> {
+  async hash({preserveOrder = false, hashFn = 'sha256'}):Promise<string> {
     if (!this.hashValue) {
-      this.hashValue = await Crypto.sha256(
-        JSON.stringify({ title: this.title, url: this.url })
-      )
+      this.hashValue = {}
+      const json = JSON.stringify({ title: this.title, url: this.url })
+      if (hashFn === 'sha256') {
+        this.hashValue[hashFn] = await Crypto.sha256(json)
+      } else if (hashFn === 'murmur3') {
+        this.hashValue[hashFn] = await Crypto.murmurHash3(json)
+      } else {
+        throw new Error('Unsupported hash function specified')
+      }
     }
-    return this.hashValue
+    return this.hashValue[hashFn]
   }
 
   clone(withHash?: boolean):Bookmark<L> {
@@ -119,6 +125,7 @@ export class Bookmark<L extends TItemLocation> {
   toJSON() {
     // Flatten inherited properties for serialization
     const result = {}
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
     let obj = this
     while (obj) {
       Object.entries(obj).forEach(([key, value]) => {
@@ -306,9 +313,10 @@ export class Folder<L extends TItemLocation> {
     return 0
   }
 
-  async hash(preserveOrder = false): Promise<string> {
-    if (this.hashValue && typeof this.hashValue[String(preserveOrder)] !== 'undefined') {
-      return this.hashValue[String(preserveOrder)]
+  async hash({preserveOrder = false, hashFn = 'sha256'}: IHashSettings): Promise<string> {
+    const cacheKey = `${preserveOrder}-${hashFn}`
+    if (this.hashValue && typeof this.hashValue[cacheKey] !== 'undefined') {
+      return this.hashValue[cacheKey]
     }
 
     if (!this.loaded) {
@@ -329,17 +337,22 @@ export class Folder<L extends TItemLocation> {
       })
     }
     if (!this.hashValue) this.hashValue = {}
-    this.hashValue[String(preserveOrder)] = await Crypto.sha256(
-      JSON.stringify({
-        title: this.title,
-        children: await Parallel.map(
-          children,
-          child => child.hash(preserveOrder),
-          1
-        )
-      })
-    )
-    return this.hashValue[String(preserveOrder)]
+    const json = JSON.stringify({
+      title: this.title,
+      children: await Parallel.map(
+        children,
+        child => child.hash({preserveOrder, hashFn}),
+        1
+      )
+    })
+    if (hashFn === 'sha256') {
+      this.hashValue[cacheKey] = await Crypto.sha256(json)
+    } else if (hashFn === 'murmur3') {
+      this.hashValue[cacheKey] = await Crypto.murmurHash3(json)
+    } else {
+      throw new Error('Unsupported hash function specified')
+    }
+    return this.hashValue[cacheKey]
   }
 
   copy(withHash?:boolean):Folder<L> {

src/lib/adapters/Caching.ts

@@ -12,13 +12,14 @@ import {
   UnknownMoveOriginError,
   UnknownMoveTargetError
 } from '../../errors/Error'
-import { BulkImportResource } from '../interfaces/Resource'
+import { BulkImportResource, ICapabilities, IHashSettings } from '../interfaces/Resource'
 
 export default class CachingAdapter implements Adapter, BulkImportResource<TItemLocation> {
   protected highestId: number
   public bookmarksCache: Folder<TItemLocation>
   protected server: any
   protected location: TItemLocation = ItemLocation.SERVER
+  protected hashSettings: IHashSettings
 
   constructor(server: any) {
     this.resetCache()
@@ -245,4 +246,15 @@ export default class CachingAdapter implements Adapter, BulkImportResource<TItem
   isAvailable(): Promise<boolean> {
     return Promise.resolve(true)
   }
+
+  async getCapabilities(): Promise<ICapabilities> {
+    return {
+      preserveOrder: true,
+      hashFn: ['murmur3', 'sha256'],
+    }
+  }
+
+  setHashSettings(hashSettings: IHashSettings): void {
+    this.hashSettings = hashSettings
+  }
 }

src/lib/adapters/Git.ts

@@ -170,7 +170,7 @@ export default class GitAdapter extends CachingAdapter {
 
     const status = await this.pullFromServer()
 
-    this.initialTreeHash = await this.bookmarksCache.hash(true)
+    this.initialTreeHash = await this.bookmarksCache.hash(this.hashSettings)
 
     Logger.log('onSyncStart: completed')
 
@@ -189,7 +189,7 @@ export default class GitAdapter extends CachingAdapter {
     clearInterval(this.lockingInterval)
 
     this.bookmarksCache = this.bookmarksCache.clone(false)
-    const newTreeHash = await this.bookmarksCache.hash(true)
+    const newTreeHash = await this.bookmarksCache.hash(this.hashSettings)
     if (newTreeHash !== this.initialTreeHash) {
       const fileContents = this.server.bookmark_file_type === 'xbel' ? createXBEL(this.bookmarksCache, this.highestId) : createHTML(this.bookmarksCache, this.highestId)
       Logger.log('(git) writeFile ' + this.dir + '/' + this.server.bookmark_file)

src/lib/adapters/GoogleDrive.ts

@@ -298,7 +298,7 @@ export default class GoogleDriveAdapter extends CachingAdapter {
       this.alwaysUpload = true
     }
 
-    this.initialTreeHash = await this.bookmarksCache.hash(true)
+    this.initialTreeHash = await this.bookmarksCache.hash(this.hashSettings)
 
     Logger.log('onSyncStart: completed')
 
@@ -335,7 +335,7 @@ export default class GoogleDriveAdapter extends CachingAdapter {
     clearInterval(this.lockingInterval)
 
     this.bookmarksCache = this.bookmarksCache.clone(false)
-    const newTreeHash = await this.bookmarksCache.hash(true)
+    const newTreeHash = await this.bookmarksCache.hash(this.hashSettings)
 
     if (!this.fileId) {
       await this.createFile(await this.getXBELContent())

src/lib/adapters/Karakeep.ts

@@ -1,7 +1,7 @@
 import Adapter from '../interfaces/Adapter'
 import { Bookmark, Folder, ItemLocation } from '../Tree'
 import PQueue from 'p-queue'
-import { IResource } from '../interfaces/Resource'
+import { ICapabilities, IResource } from '../interfaces/Resource'
 import Logger from '../Logger'
 import {
   AuthenticationError,
@@ -565,4 +565,11 @@ export default class KarakeepAdapter implements Adapter, IResource<typeof ItemLo
 
     return json
   }
+
+  async getCapabilities(): Promise<ICapabilities> {
+    return {
+      preserveOrder: false,
+      hashFn: ['murmur3', 'sha256'],
+    }
+  }
 }

src/lib/adapters/Linkwarden.ts

@@ -1,7 +1,7 @@
 import Adapter from '../interfaces/Adapter'
 import { Bookmark, Folder, ItemLocation } from '../Tree'
 import PQueue from 'p-queue'
-import { IResource } from '../interfaces/Resource'
+import { ICapabilities, IResource } from '../interfaces/Resource'
 import Logger from '../Logger'
 import {
   AuthenticationError,
@@ -367,4 +367,11 @@ export default class LinkwardenAdapter implements Adapter, IResource<typeof Item
 
     return json
   }
+
+  async getCapabilities(): Promise<ICapabilities> {
+    return {
+      preserveOrder: false,
+      hashFn: ['murmur3', 'sha256'],
+    }
+  }
 }

src/lib/adapters/NextcloudBookmarks.ts

@@ -1,5 +1,4 @@
 // Nextcloud ADAPTER
-// All owncloud specifc stuff goes in here
 import { Capacitor, CapacitorHttp as Http } from '@capacitor/core'
 import Adapter from '../interfaces/Adapter'
 import HtmlSerializer from '../serializers/Html'
@@ -12,7 +11,7 @@ import PQueue from 'p-queue'
 import flatten from 'lodash/flatten'
 import {
   BulkImportResource,
-  ClickCountResource,
+  ClickCountResource, ICapabilities, IHashSettings,
   LoadFolderChildrenResource,
   OrderFolderResource
 } from '../interfaces/Resource'
@@ -78,6 +77,7 @@ export default class NextcloudBookmarksAdapter implements Adapter, BulkImportRes
   private ended = false
   private locked = false
   private hasFeatureJavascriptLinks: boolean = null
+  private hashSettings: IHashSettings
 
   constructor(server: NextcloudBookmarksConfig) {
     this.server = server
@@ -316,6 +316,9 @@ export default class NextcloudBookmarksAdapter implements Adapter, BulkImportRes
   }
 
   async _getFolderHash(folderId:string|number):Promise<string> {
+    if (this.hashSettings.hashFn !== 'sha256') {
+      throw new Error('Unsupported hash function: ' + this.hashSettings.hashFn + ' - Nextcloud Bookmarks only supports sha256')
+    }
     return this.sendRequest(
       'GET',
       `index.php/apps/bookmarks/public/rest/v2/folder/${folderId}/hash`
@@ -363,7 +366,7 @@ export default class NextcloudBookmarksAdapter implements Adapter, BulkImportRes
     return recurseChildren(folderId, children)
   }
 
-  async loadFolderChildren(folderId:string|number, all?: boolean): Promise<TItem<typeof ItemLocation.SERVER>[]> {
+  async loadFolderChildren(folderId:string|number, hashSettings: IHashSettings, all?: boolean): Promise<TItem<typeof ItemLocation.SERVER>[]> {
     const folder = this.tree.findFolder(folderId)
     if (!folder) {
       throw new Error('Could not find folder for loadFolderChildren')
@@ -977,6 +980,13 @@ export default class NextcloudBookmarksAdapter implements Adapter, BulkImportRes
     return Promise.resolve(true)
   }
 
+  async getCapabilities(): Promise<ICapabilities> {
+    return {
+      preserveOrder: true,
+      hashFn: ['sha256'],
+    }
+  }
+
   async countClick(url: string): Promise<void> {
     try {
       await this.sendRequest(
@@ -991,4 +1001,8 @@ export default class NextcloudBookmarksAdapter implements Adapter, BulkImportRes
       console.warn(e)
     }
   }
+
+  setHashSettings(hashSettings: IHashSettings): void {
+    this.hashSettings = hashSettings
+  }
 }