Merge pull request certinia#612 from lukecotter/perf-apex-log-parser

perf: improve apex log parser speed

Author: lcottercertinia

log-viewer/modules/components/LogViewer.ts

@@ -97,38 +97,41 @@ export class LogViewer extends LitElement {
     this.notifications = localNotifications;
 
     this.parserIssues = this.parserIssuesToMessages(apexLog);
-
     this.logStatus = 'Ready';
   }
 
   async _readLog(logUri: string): Promise<string> {
+    let msg = '';
     if (logUri) {
-      return fetch(logUri)
-        .then((response) => {
-          if (response.ok) {
-            return response.text();
-          } else {
-            throw Error(response.statusText || `Error reading log file: ${response.status}`);
+      try {
+        const response = await fetch(logUri);
+        if (!response.ok || !response.body) {
+          throw new Error(response.statusText || `Error reading log file: ${response.status}`);
+        }
+
+        const reader = response.body.pipeThrough(new TextDecoderStream()).getReader();
+        const chunks: string[] = [];
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) {
+            break;
           }
-        })
-        .catch((err: unknown) => {
-          const msg = err instanceof Error ? err.message : String(err);
-
-          const logMessage = new Notification();
-          logMessage.summary = 'Could not read log';
-          logMessage.message = msg || '';
-          logMessage.severity = 'Error';
-          this.notifications.push(logMessage);
-          return Promise.resolve('');
-        });
+          chunks.push(value);
+        }
+        return chunks.join('');
+      } catch (err: unknown) {
+        msg = (err instanceof Error ? err.message : String(err)) ?? '';
+      }
     } else {
-      const logMessage = new Notification();
-      logMessage.summary = 'Could not read log';
-      logMessage.message = 'Invalid Log Path';
-      logMessage.severity = 'Error';
-      this.notifications.push(logMessage);
-      return Promise.resolve('');
+      msg = 'Invalid Log Path';
     }
+
+    const logMessage = new Notification();
+    logMessage.summary = 'Could not read log';
+    logMessage.message = msg;
+    logMessage.severity = 'Error';
+    this.notifications.push(logMessage);
+    return '';
   }
 
   severity = new Map<string, NotificationSeverity>([

log-viewer/modules/parsers/ApexLogParser.ts

@@ -42,8 +42,7 @@ export class ApexLogParser {
   cpuUsed = 0;
   lastTimestamp = 0;
   discontinuity = false;
-  namespaces: string[] = [];
-  namespacesUniq = new Set<string>();
+  namespaces = new Set<string>();
 
   /**
    * Takes string input of a log and returns the ApexLog class, which represents a log tree
@@ -58,7 +57,7 @@ export class ApexLogParser {
     apexLog.logIssues = this.logIssues;
     apexLog.parsingErrors = this.parsingErrors;
     apexLog.cpuTime = this.cpuUsed;
-    apexLog.namespaces = this.namespaces;
+    apexLog.namespaces = Array.from(this.namespaces);
 
     return apexLog;
   }
@@ -67,19 +66,19 @@ export class ApexLogParser {
     const parts = line.split('|');
 
     const type = parts[1] ?? '';
+
     const metaCtor = getLogEventClass(type as LogEventType);
     if (metaCtor) {
       const entry = new metaCtor(this, parts);
       entry.logLine = line;
       lastEntry?.onAfter?.(this, entry);
-      if (entry.namespace && !this.namespacesUniq.has(entry.namespace)) {
-        this.namespaces.push(entry.namespace);
-        this.namespacesUniq.add(entry.namespace);
+      if (entry.namespace) {
+        this.namespaces.add(entry.namespace);
       }
       return entry;
     }
 
-    const hasType = type && typePattern.test(type);
+    const hasType = !!(type && typePattern.test(type));
     if (!hasType && lastEntry?.acceptsText) {
       // wrapped text from the previous entry?
       lastEntry.text += '\n' + line;
@@ -169,11 +168,10 @@ export class ApexLogParser {
       line.parent = rootMethod;
       rootMethod.addChild(line);
     }
-    rootMethod.setTimes();
 
+    rootMethod.setTimes();
     this.insertPackageWrappers(rootMethod);
     this.aggregateTotals([rootMethod]);
-
     return rootMethod;
   }
 
@@ -220,40 +218,39 @@ export class ApexLogParser {
           break;
         }
 
-        nextLine.namespace ||= currentLine.namespace || 'default';
         lineIter.fetch(); // it's a child - consume the line
         this.lastTimestamp = nextLine.timestamp;
+        nextLine.namespace ||= currentLine.namespace || 'default';
+        nextLine.parent = currentLine;
+        currentLine.children.push(nextLine);
 
         if (nextLine instanceof Method) {
           this.parseTree(nextLine, lineIter, stack);
         }
-
-        nextLine.parent = currentLine;
-        currentLine.children.push(nextLine);
       }
 
       // End of line error handling. We have finished processing this log line and either got to the end
       // of the log without finding an exit line or the current line was truncated)
       if (!nextLine || currentLine.isTruncated) {
         // truncated method - terminate at the end of the log
-        currentLine.exitStamp = this.lastTimestamp;
+        currentLine.exitStamp = this.lastTimestamp ?? currentLine.timestamp;
 
         // we found an entry event on its own e.g a `METHOD_ENTRY` without a `METHOD_EXIT` and got to the end of the log
         this.addLogIssue(
-          this.lastTimestamp,
+          currentLine.exitStamp,
           'Unexpected-End',
           'An entry event was found without a corresponding exit event e.g a `METHOD_ENTRY` event without a `METHOD_EXIT`',
           'unexpected',
         );
 
         if (currentLine.isTruncated) {
           this.updateLogIssue(
-            this.lastTimestamp,
+            currentLine.exitStamp,
             'Max-Size-reached',
             'The maximum log size has been reached. Part of the log has been truncated.',
             'skip',
           );
-          this.maxSizeTimestamp = this.lastTimestamp;
+          this.maxSizeTimestamp = currentLine.exitStamp;
         }
         currentLine.isTruncated = true;
       }
@@ -264,7 +261,7 @@ export class ApexLogParser {
   }
 
   private isMatchingEnd(startMethod: Method, endLine: LogLine) {
-    return (
+    return !!(
       endLine.type &&
       startMethod.exitTypes.includes(endLine.type) &&
       (endLine.lineNumber === startMethod.lineNumber ||
@@ -305,26 +302,26 @@ export class ApexLogParser {
 
   private flattenByDepth(nodes: LogLine[]) {
     const result = new Map<number, LogLine[]>();
-    result.set(0, nodes);
-
-    let currentDepth = 1;
 
+    let currentDepth = 0;
     let currentNodes = nodes;
     let len = currentNodes.length;
     while (len) {
-      result.set(currentDepth, []);
+      result.set(currentDepth, currentNodes);
+
+      const children: LogLine[] = [];
       while (len--) {
         const node = currentNodes[len];
         if (node?.children) {
-          const children = result.get(currentDepth)!;
           node.children.forEach((c) => {
             if (c.children.length) {
               children.push(c);
             }
           });
         }
       }
-      currentNodes = result.get(currentDepth++) ?? [];
+      currentDepth++;
+      currentNodes = children;
       len = currentNodes.length;
     }
 
@@ -677,7 +674,7 @@ export abstract class LogLine {
     if (parts) {
       const [timeData, type] = parts;
       this.text = this.type = type as LogEventType;
-      this.timestamp = this.parseTimestamp(timeData || '');
+      this.timestamp = timeData ? this.parseTimestamp(timeData) : 0;
     } else {
       this.timestamp = 0;
       this.text = '';
@@ -965,14 +962,10 @@ class ConstructorEntryLine extends Method {
 
   _parseConstructorNamespace(className: string): string {
     let possibleNs = className.slice(0, className.indexOf('.'));
-    possibleNs =
-      this.logParser.namespaces.find((ns) => {
-        return ns === possibleNs;
-      }) || '';
-
-    if (possibleNs) {
+    if (this.logParser.namespaces.has(possibleNs)) {
       return possibleNs;
     }
+
     const constructorParts = (className ?? '').split('.');
     possibleNs = constructorParts[0] || '';
     // inmner class with a namespace
@@ -1007,12 +1000,12 @@ export class MethodEntryLine extends Method {
   constructor(parser: ApexLogParser, parts: string[]) {
     super(parser, parts, ['METHOD_EXIT'], 'Method', 'method');
     this.lineNumber = this.parseLineNumber(parts[2]);
-    this.text = parts[4] || this.type || '';
+    this.text = parts[4] || this.type || this.text;
     if (this.text.indexOf('System.Type.forName(') !== -1) {
       // assume we are not charged for class loading (or at least not lengthy remote-loading / compiling)
       this.cpuType = 'loading';
     } else {
-      const possibleNs = this._parseMethodNamespace(parts[4] || '');
+      const possibleNs = this._parseMethodNamespace(parts[4]);
       if (possibleNs) {
         this.namespace = possibleNs;
       }
@@ -1025,23 +1018,27 @@ export class MethodEntryLine extends Method {
     }
   }
 
-  _parseMethodNamespace(methodName: string): string {
+  _parseMethodNamespace(methodName: string | undefined): string {
+    if (!methodName) {
+      return '';
+    }
+
     const methodBracketIndex = methodName.indexOf('(');
     if (methodBracketIndex === -1) {
       return '';
     }
 
-    let possibleNs = methodName.slice(0, methodName.indexOf('.'));
-    possibleNs =
-      this.logParser.namespaces.find((ns) => {
-        return ns === possibleNs;
-      }) || '';
+    const nsSeparator = methodName.indexOf('.');
+    if (nsSeparator === -1) {
+      return '';
+    }
 
-    if (possibleNs) {
+    const possibleNs = methodName.slice(0, nsSeparator);
+    if (this.logParser.namespaces.has(possibleNs)) {
       return possibleNs;
     }
 
-    const methodNameParts = methodName ? methodName.slice(0, methodBracketIndex)?.split('.') : '';
+    const methodNameParts = methodName.slice(0, methodBracketIndex)?.split('.');
     if (methodNameParts.length === 4) {
       return methodNameParts[0] ?? '';
     } else if (methodNameParts.length === 2) {
@@ -1057,9 +1054,12 @@ class MethodExitLine extends LogLine {
   constructor(parser: ApexLogParser, parts: string[]) {
     super(parser, parts);
     this.lineNumber = this.parseLineNumber(parts[2]);
-    this.text = parts[4] ?? parts[3] ?? '';
+    this.text = parts[4] ?? parts[3] ?? this.text;
 
+    /*A method will end with ')'. Without that this it represents the first reference to a class, outer or inner. One of the few reliable ways to determine valid namespaces. The first reference to a class (outer or inner) will always have an METHOD_EXIT containing the Outer class name with namespace if present. Other events will follow, CONSTRUCTOR_ENTRY etc. But this case will only ever have 2 parts ns.Outer even if the first reference was actually an inner class e.g new ns.Outer.Inner();*/
+    // If does not end in ) then we have a reference to the class, either via outer or inner.
     if (!this.text.endsWith(')')) {
+      // if there is a . the we have a namespace e.g ns.Outer
       const index = this.text.indexOf('.');
       if (index !== -1) {
         this.namespace = this.text.slice(0, index);