feat(): lru memoization for js worker

lib/memoize/index.ts

@@ -0,0 +1,98 @@
+import LRUCache from 'lru-cache';
+import Crypto from '../utils/crypto';
+
+/**
+ * Pick the strategy of cache key form
+ * It could be concatenated list of arguments like 'projectId:eventId'
+ * Or it could be hashed json object — blake2b512 algorithm
+ */
+export type MemoizeKeyStrategy = 'concat' | 'hash';
+
+/**
+ * Options of the memoize decorator
+ */
+export interface MemoizeOptions {
+  /**
+   * Max number of values stored in LRU cache at the same time
+   */
+  max?: number;
+
+  /**
+   * TTL in milliseconds
+   */
+  ttl?: number;
+
+  /**
+   * Strategy for key generation
+   */
+  strategy?: MemoizeKeyStrategy;
+}
+
+/**
+ * Async-only, per-method LRU-backed memoization decorator.
+ * Cache persists for the lifetime of the class instance (e.g. worker).
+ *
+ * @param options
+ */
+export function memoize(options: MemoizeOptions = {}): MethodDecorator {
+  /* eslint-disable @typescript-eslint/no-magic-numbers */
+  const {
+    max = 50,
+    ttl = 1000 * 60 * 30,
+    strategy = 'concat',
+  } = options;
+  /* eslint-enable */
+
+  return function (
+    _target,
+    propertyKey,
+    descriptor: PropertyDescriptor
+  ): PropertyDescriptor {
+    const originalMethod = descriptor.value;
+
+    if (typeof originalMethod !== 'function') {
+      throw new Error('@Memoize can only decorate methods');
+    }
+
+    descriptor.value = async function (...args: unknown[]): Promise<unknown> {
+      /**
+       * Create a cache key for each decorated method
+       */
+      const cacheKey = `memoizeCache:${String(propertyKey)}`;
+
+      /**
+       * Create a new cache if it does not exists yet (for certain function)
+       */
+      const cache: LRUCache<string, any> = this[cacheKey] ??= new LRUCache<string, any>({
+        max,
+        ttl,
+      });
+
+      const key = strategy === 'hash'
+        ? Crypto.hash(args, 'blake2b512', 'base64url')
+        : args.map(String).join('__ARG_JOIN__');
+
+      /**
+       * Check if we have a cached result
+       */
+      const cachedResult = cache.get(key);
+
+      if (cachedResult !== undefined) {
+        return cachedResult;
+      }
+
+      try {
+        const result = await originalMethod.apply(this, args);
+
+        cache.set(key, result);
+
+        return result;
+      } catch (err) {
+        cache.delete(key);
+        throw err;
+      }
+    };
+
+    return descriptor;
+  };
+}

lib/utils/crypto.ts

@@ -1,4 +1,4 @@
-import crypto from 'crypto';
+import crypto, { BinaryToTextEncoding } from 'crypto';
 
 /**
  * Crypto helper
@@ -9,12 +9,13 @@ export default class Crypto {
    *
    * @param value — data to be hashed
    * @param algo — type of algorithm to be used for hashing
+   * @param digest - type of the representation of the hashed value
    */
-  public static hash(value: unknown, algo = 'sha256'): string {
-    const stringifiedValue = JSON.stringify(value);
+  public static hash(value: unknown, algo = 'sha256', digest: BinaryToTextEncoding = 'hex'): string {
+    const stringifiedValue = typeof value === 'string' ? value : JSON.stringify(value);
 
     return crypto.createHash(algo)
       .update(stringifiedValue)
-      .digest('hex');
+      .digest(digest);
   }
 }

tsconfig.json

@@ -58,7 +58,7 @@
         // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
 
         /* Experimental Options */
-        // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
+        "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
         // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
 
         /* Advanced Options */

workers/javascript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hawk-worker-javascript",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "description": "Handles messages from JavaScript Catcher",
   "main": "src/index.ts",
   "license": "UNLICENSED",
@@ -10,7 +10,8 @@
     "@types/useragent": "^2.1.1",
     "source-map-js": "^1.2.0",
     "ts-node": "^8.3.0",
-    "typescript": "^3.5.3"
+    "typescript": "^3.5.3",
+    "lodash.clonedeep": "^4.5.0"
   },
   "dependencies": {
     "useragent": "^2.3.0"

workers/javascript/src/index.ts

@@ -7,13 +7,21 @@ import { GroupWorkerTask } from '../../grouper/types/group-worker-task';
 import { SourceMapsRecord } from '../../release/types';
 import * as pkg from '../package.json';
 import { JavaScriptEventWorkerTask } from '../types/javascript-event-worker-task';
+import { BeautifyBacktracePayload } from '../types/beautify-backtrace-payload';
 import HawkCatcher from '@hawk.so/nodejs';
-import Crypto from '../../../lib/utils/crypto';
 import { BacktraceFrame, CatcherMessagePayload, CatcherMessageType, ErrorsCatcherType, SourceCodeLine, SourceMapDataExtended } from '@hawk.so/types';
 import { beautifyUserAgent } from './utils';
 import { Collection } from 'mongodb';
 import { parse } from '@babel/parser';
 import traverse from '@babel/traverse';
+/* eslint-disable-next-line no-unused-vars */
+import { memoize } from '../../../lib/memoize';
+
+/**
+ * eslint does not count decorators as a variable usage
+ */
+/* eslint-disable-next-line no-unused-vars */
+const MEMOIZATION_TTL = Number(process.env.MEMOIZATION_TTL ?? 0);
 
 /**
  * Worker for handling Javascript events
@@ -69,7 +77,11 @@ export default class JavascriptEventWorker extends EventWorker {
       this.logger.info('beautifyBacktrace called');
 
       try {
-        event.payload.backtrace = await this.beautifyBacktrace(event);
+        event.payload.backtrace = await this.beautifyBacktrace({
+          projectId: event.projectId,
+          release: event.payload.release.toString(),
+          backtrace: event.payload.backtrace,
+        });
       } catch (err) {
         this.logger.error('Error while beautifing backtrace', err);
       }
@@ -94,52 +106,38 @@ export default class JavascriptEventWorker extends EventWorker {
    * @param {JavaScriptEventWorkerTask} event — js error minified
    * @returns {BacktraceFrame[]} - parsed backtrace
    */
-  private async beautifyBacktrace(event: JavaScriptEventWorkerTask): Promise<BacktraceFrame[]> {
-    const releaseRecord: SourceMapsRecord = await this.cache.get(
-      `releaseRecord:${event.projectId}:${event.payload.release.toString()}`,
-      () => {
-        return this.getReleaseRecord(
-          event.projectId,
-          event.payload.release.toString()
-        );
-      }
-    );
+  @memoize({ max: 200, ttl: MEMOIZATION_TTL, strategy: 'hash' })
+  private async beautifyBacktrace({ projectId, release, backtrace }: BeautifyBacktracePayload): Promise<BacktraceFrame[]> {
+    const releaseRecord: SourceMapsRecord = await this.getReleaseRecord(projectId, release);
 
     if (!releaseRecord) {
       this.logger.info('beautifyBacktrace: no releaseRecord found');
 
-      return event.payload.backtrace;
+      return backtrace;
     }
 
     this.logger.info(`beautifyBacktrace: release record found: ${JSON.stringify(releaseRecord)}`);
 
     /**
      * If we have a source map associated with passed release, override some values in backtrace with original line/file
      */
-    return Promise.all(event.payload.backtrace.map(async (frame: BacktraceFrame, index: number) => {
+    return Promise.all(backtrace.map(async (frame: BacktraceFrame, index: number) => {
       /**
-       * Get cached (or set if the value is missing) real backtrace frame
+       * Consume rbacktrace frame and catch errors (send them to hawk)
        */
-      const result = await this.cache.get(
-        `consumeBacktraceFrame:${event.payload.release.toString()}:${Crypto.hash(frame)}:${index}`,
-        () => {
-          return this.consumeBacktraceFrame(frame, releaseRecord)
-            .catch((error) => {
-              this.logger.error('Error while consuming ' + error.stack);
-
-              /**
-               * Send error to Hawk
-               */
-              HawkCatcher.send(error, {
-                payload: event.payload as unknown as Record<string, never>,
-              });
-
-              return event.payload.backtrace[index];
-            });
-        }
-      );
-
-      return result;
+      return await this.consumeBacktraceFrame(frame, releaseRecord)
+        .catch((error) => {
+          this.logger.error('Error while consuming ' + error.stack);
+
+          /**
+           * Send error to Hawk
+           */
+          HawkCatcher.send(error, {
+            payload: backtrace as unknown as Record<string, never>,
+          });
+
+          return backtrace[index];
+        });
     }));
   }
 
@@ -189,17 +187,14 @@ export default class JavascriptEventWorker extends EventWorker {
     /**
      * Load source map content from Grid fs
      */
-    const mapContent = await this.loadSourceMapFile(mapForFrame);
+    const mapContent = await this.loadSourceMapFile(mapForFrame._id);
 
     if (!mapContent) {
       this.logger.info(`consumeBacktraceFrame: Can't load map content for ${JSON.stringify(mapForFrame)}`);
 
       return stackFrame;
     }
 
-    /**
-     * @todo cache source map consumer for file-keys
-     */
     const consumer = this.consumeSourceMap(mapContent);
 
     /**
@@ -234,10 +229,10 @@ export default class JavascriptEventWorker extends EventWorker {
          */
         lines = this.readSourceLines(consumer, originalLocation);
 
-    //     const originalContent = consumer.sourceContentFor(originalLocation.source);
+        const originalContent = consumer.sourceContentFor(originalLocation.source);
 
-    //     functionContext = this.getFunctionContext(originalContent, originalLocation.line) ?? originalLocation.name;
-      } catch(e) {
+        functionContext = await this.getFunctionContext(originalContent, originalLocation.line) ?? originalLocation.name;
+      } catch (e) {
         HawkCatcher.send(e);
         this.logger.error('Can\'t get function context');
         this.logger.error(e);
@@ -260,7 +255,7 @@ export default class JavascriptEventWorker extends EventWorker {
    * @param line - number of the line from the stack trace
    * @returns {string | null} - string of the function context or null if it could not be parsed
    */
-  private _getFunctionContext(sourceCode: string, line: number): string | null {
+  private getFunctionContext(sourceCode: string, line: number): string | null {
     let functionName: string | null = null;
     let className: string | null = null;
     let isAsync = false;
@@ -361,13 +356,13 @@ export default class JavascriptEventWorker extends EventWorker {
   /**
    * Downloads source map file from Grid FS
    *
-   * @param map - saved file info without content.
+   * @param mapId - id of the map file in the bucket
    */
-  private loadSourceMapFile(map: SourceMapDataExtended): Promise<string> {
+  private loadSourceMapFile(mapId: SourceMapDataExtended['_id']): Promise<string> {
     return new Promise((resolve, reject) => {
       let buf = Buffer.from('');
 
-      const readstream = this.db.getBucket().openDownloadStream(map._id)
+      const readstream = this.db.getBucket().openDownloadStream(mapId)
         .on('data', (chunk) => {
           buf = Buffer.concat([buf, chunk]);
         })