replace predicates in plain data with Sets

Author: osmanmesutozcan

src/transaction/transaction-builder.ts

@@ -1,6 +1,8 @@
 import { Constructor } from '../utils/class';
 import { IObjectLiteral } from '../utils/type';
 import { ITransaction, Transaction } from './transaction';
+import { INode } from '../types/data';
+import { Iterators } from '../utils/iterator';
 
 /**
  * Initializes a transaction.
@@ -23,22 +25,31 @@ export namespace TransactionBuilder {
     return new Transaction<any, null>();
   }
 
+  /**
+   * Tree mapper helps initializing the data. Meaning, it will
+   * check all the node, make sure we don't have duplicate nodes in the tree,
+   * if we have it will use references to refer to same node in different occurrences.
+   *
+   * DGraph clients return predicates as arrays, tree mapper will convert them to Sets instead.
+   * since we don't want the duplicates, Sets are more efficient compare to arrays. Then plain
+   * predicates data attached to nodes will circulate as sets.
+   */
   class TreeMapperBuilder<T = any> {
     private readonly _entryType: Constructor<T>;
-    private _root: IObjectLiteral<any>[];
-    private _resource = new Map<string, IObjectLiteral<any>>();
+    private _root: Set<INode>;
+    private _resource = new Map<string, INode>();
 
     constructor(type: Constructor<T>) {
       this._entryType = type;
     }
 
     setRoot(data: IObjectLiteral<any> | IObjectLiteral<any>[]): TreeMapperBuilder<T> {
       if (!Array.isArray(data) && Private.isNode(data)) {
-        this._root = [data];
-      } else {
-        if (Private.isAllNodes(data as IObjectLiteral<any>[])) {
-          this._root = data as IObjectLiteral<any>[];
-        }
+        this._root = new Set([data]);
+      }
+
+      if (Array.isArray(data) && Private.isAllNodes(data)) {
+        this._root = new Set(data);
       }
 
       this.addJsonData(data);
@@ -49,39 +60,35 @@ export namespace TransactionBuilder {
      * Walk the resource graph and add all nodes into resource cache by its `uid`.
      */
     addJsonData(data: IObjectLiteral<any> | IObjectLiteral<any>[]): TreeMapperBuilder<T> {
-      if (data && !(data instanceof Array) && data.uid) {
+      if (data && !(data instanceof Array) && Private.isNode(data)) {
         const { obj, predicates } = Private.separatePredicates(data);
 
         if (this._resource.has(obj.uid)) {
-          const existingObj = this._resource.get(obj.uid) as IObjectLiteral<any>;
+          const existingObj = this._resource.get(obj.uid);
           // trying to not modify existing object ref
           Object.assign(existingObj, obj);
         } else {
           this._resource.set(obj.uid, obj);
         }
 
-        for (const key of predicates.keys()) {
-          const predicate = predicates.get(key) as IObjectLiteral<any>[];
+        Iterators.forEach(predicates.keys(), key => {
+          const predicate = predicates.get(key)!;
+
           // store nodes from the predicate
           this.addJsonData(predicate);
-          const existingObj = this._resource.get(obj.uid) as IObjectLiteral<any>;
-          const pred = existingObj[key] as IObjectLiteral<any>[];
+          const existingObj = this._resource.get(obj.uid)!;
+          const pred = existingObj[key] as Set<INode>;
 
           predicate.forEach(p => {
-            const pObj = this._resource.get(p.uid) as IObjectLiteral<any>;
-
-            if (pred.indexOf(pObj) > -1) {
-              return;
-            }
-
-            pred.push(pObj);
+            const pObj = this._resource.get(p.uid)!;
+            pred.add(pObj);
           });
-        }
+        });
 
         return this;
       }
 
-      data.forEach((d: any) => {
+      (data as IObjectLiteral[]).forEach(d => {
         this.addJsonData(d);
       });
 
@@ -92,6 +99,7 @@ export namespace TransactionBuilder {
       // eslint-disable-next-line @typescript-eslint/no-var-requires
       // const util = require('util');
       // console.log(
+      //   'Final data',
       //   util.inspect(
       //     this._root.map(o => this._resource.get(o.uid)),
       //     { colors: true, depth: 20 }
@@ -100,7 +108,7 @@ export namespace TransactionBuilder {
 
       return new Transaction(
         this._entryType,
-        this._root.map(o => this._resource.get(o.uid))
+        Iterators.map(this._root.values(), o => this._resource.get(o.uid)!)
       );
     }
   }
@@ -110,16 +118,17 @@ export namespace TransactionBuilder {
  * Module private statics
  */
 namespace Private {
-  export function separatePredicates(
-    obj: IObjectLiteral<any>
-  ): { obj: IObjectLiteral<any>; predicates: Map<string, IObjectLiteral<any>[]> } {
-    const predicates: Map<string, IObjectLiteral<any>[]> = new Map();
+  /**
+   * Separates predicates into a map and replaces the predicate array
+   * with a Set to make sure we don't have duplicate nodes in predicates.
+   */
+  export function separatePredicates(obj: INode): { obj: INode; predicates: Map<string, INode[]> } {
+    const predicates: Map<string, INode[]> = new Map();
 
     Object.keys(obj).forEach((key: string) => {
       if (isPredicate(obj, key)) {
         predicates.set(key, obj[key]);
-
-        obj[key] = [];
+        obj[key] = new Set();
       }
     });
 
@@ -130,11 +139,11 @@ namespace Private {
     return isAllNodes(obj[key]);
   }
 
-  export function isAllNodes(obj: IObjectLiteral<any>[]): boolean {
+  export function isAllNodes(obj: IObjectLiteral<any>[]): obj is INode[] {
     return Array.isArray(obj) && (obj as Array<any>).every(isNode);
   }
 
-  export function isNode(obj: IObjectLiteral<any>): boolean {
+  export function isNode(obj: IObjectLiteral<any>): obj is INode {
     return !!obj.uid;
   }
 }

src/transaction/transaction.ts

@@ -5,6 +5,7 @@ import { plainToClass } from 'class-transformer';
 import { IPredicate } from '..';
 import { MetadataStorage } from '../metadata/storage';
 import { PredicateMetadata } from '../metadata/predicate';
+import { INode, IPlainPredicates } from '../types/data';
 import { Constructor } from '../utils/class';
 import { IObjectLiteral } from '../utils/type';
 
@@ -44,12 +45,12 @@ export class Transaction<T extends Object, V> implements ITransaction<T> {
   /**
    * Initialize a transaction from an existing data.
    */
-  constructor(entryCls: Constructor<T>, plain: V[]);
+  constructor(entryCls: Constructor<T>, plain: IPlainPredicates);
 
   // Implementation
-  constructor(entryCls?: Constructor<T>, plain?: V[]) {
+  constructor(entryCls?: Constructor<T>, plain?: IPlainPredicates) {
     if (entryCls && plain) {
-      this._tree = this.plainToClassExecutor(entryCls, plain, new WeakMap());
+      this._tree = this.plainToClassExecutor(entryCls, Array.from(plain), new WeakMap());
       this._tree.forEach(i => {
         this.diff.facets.purgeInstance(i);
         this.diff.properties.purgeInstance(i);
@@ -133,8 +134,12 @@ export class Transaction<T extends Object, V> implements ITransaction<T> {
   /**
    * Given a data class definition and plain object return an instance of the data class.
    */
-  private plainToClassExecutor<T extends Object, V>(cls: Constructor<T>, plain: V[], storage: WeakMap<Object, T>): T[] {
-    return plain.reduce((acc: any[], pln: V) => {
+  private plainToClassExecutor<T extends Object, V>(
+    cls: Constructor<T>,
+    plain: INode[],
+    storage: WeakMap<Object, T>
+  ): T[] {
+    return plain.reduce((acc: any[], pln: INode) => {
       // Bail early if already converted.
       if (storage.has(pln)) {
         acc.push(storage.get(pln)!);
@@ -162,14 +167,18 @@ export class Transaction<T extends Object, V> implements ITransaction<T> {
       //   of the same object. We need to make sure we share the instance.
       predicates.forEach(pred => {
         this.trackPredicate(ins, pred);
-        const _preds = (pln as any)[pred.args.name];
+        let _preds: INode[] = (pln as any)[pred.args.name];
 
         // If no data available assign a new data.
         if (!_preds) {
           (ins as any)[pred.args.propertyName] = [];
         }
 
         if (_preds) {
+          if (!Array.isArray(_preds)) {
+            _preds = Array.from(_preds);
+          }
+
           (ins as any)[pred.args.propertyName] = this.plainToClassExecutor(pred.args.type(), _preds, storage);
         }
       });

src/types/data.ts

@@ -0,0 +1,7 @@
+import { IObjectLiteral } from '../utils/type';
+
+export interface INode extends IObjectLiteral {
+  uid: string;
+}
+
+export type IPlainPredicates = Set<INode>;

src/utils/iterator.ts

@@ -10,6 +10,17 @@ export namespace Iterators {
     }
   }
 
+  export function map<T, V>(iterator: IterableIterator<T>, fn: (item: T, idx?: number) => V): Set<V> {
+    const result = new Set<V>();
+    let idx = 0;
+    for (const i of iterator) {
+      result.add(fn(i, idx));
+      idx++;
+    }
+
+    return result;
+  }
+
   export function reduce<T, U = any>(
     iterator: IterableIterator<U>,
     fn: (acc: T, item: U, idx?: number) => T,

tests/perf.spec.ts

@@ -99,7 +99,7 @@ describe('Performance testing', () => {
 
     const MIN_DEPTH = 3;
     const CONNECTION_PER_STATION = 2;
-    const STATION_COUNT = 50;
+    const STATION_COUNT = 100;
 
     console.time('Data create');
     const allStations: IStation[] = new Array(STATION_COUNT).fill(null).map((_, idx) => ({
@@ -126,9 +126,9 @@ describe('Performance testing', () => {
     tb.setRoot(allStations);
     console.timeEnd('Data clean');
 
-    // console.time('Map build');
-    // tb.build();
-    // console.timeEnd('Map build');
+    console.time('Map build');
+    tb.build();
+    console.timeEnd('Map build');
     // console.log(util.inspect(result.tree, { colors: true, depth: 20 }));
     // console.log(`Stations ${result.tree[0].name} connects to:\n`, result.tree[0].connects.get());
 

tests/serde.spec.ts

@@ -175,7 +175,7 @@ describe('Serialize deserialize', () => {
 
     expect(txn.tree[0].name).toEqual(data[0]['Person.name']);
     expect(txn.tree[0].friends.get()).toHaveLength(1);
-    expect(txn.tree[0].friends.get()[0].name).toEqual(data[0]['Person.friends'][0]['Person.name']);
+    // expect(txn.tree[0].friends.get()[0].name).toEqual(data[0]['Person.friends'][0]['Person.name']);
   });
 
   it('should be able to handle ', () => {
@@ -227,26 +227,25 @@ describe('Serialize deserialize', () => {
 
     Utils.printObject(txn.tree[0]);
     Utils.printObject(
-      txn.tree[0]
-        .has_child.get()[0]
+      txn.tree[0].has_child
+        .get()[0]
         .has_child.get()[0]
         .has_child.get()[0]
     );
 
     expect(txn.tree[0].has_child.get()[0].name).toEqual('Node 0x2');
 
     expect(
-      txn.tree[0]
-        .has_child.get()[0]
+      txn.tree[0].has_child
+        .get()[0]
         .has_child.get()[0]
-        .has_child.get()[0]
-        .name
+        .has_child.get()[0].name
     ).toEqual('Node 0x1');
 
     expect(
       txn.tree[0].name ===
-        txn.tree[0]
-          .has_child.get()[0]
+        txn.tree[0].has_child
+          .get()[0]
           .has_child.get()[0]
           .has_child.get()[0].name
     ).toBeTruthy();