Ignore logs based on release version.

dleclercpro · dleclercpro · commit 3ca568c614db · 2023-10-04T15:39:06.000+02:00
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "berlin-lea-performance-monitor",
-  "version": "1.8.1",
+  "version": "1.9.0",
   "author": "David Leclerc",
   "main": "./src/index.ts",
   "scripts": {
diff --git a/src/config/index.ts b/src/config/index.ts
@@ -6,7 +6,7 @@ import Release from '../models/Release';
 
 export const ENV = loadEnvironment();
 
-export const MINIMUM_RELEASE = new Release(0, 0, 0);
+export const RELEASE_MIN = new Release(0, 0, 0);
 export const IGNORE_DAYS_WITH_EMPTY_BUCKETS = false;
 
 export const HOMEPAGE_URL = 'http://otv.verwalt-berlin.de/';
diff --git a/src/index.ts b/src/index.ts
@@ -17,6 +17,7 @@ import { LOGS_DIR, LOGS_FILEPATH } from './config/file';
 import { parseLogs } from './utils/parsing';
 import SessionHistoryBuilder from './models/sessions/SessionHistoryBuilder';
 import SessionHistoryExporter from './models/sessions/SessionHistoryExporter';
+import Release from './models/Release';
 
 
 
@@ -53,7 +54,7 @@ const execute = async () => {
         const now = new Date();
         const lastWeek = computeDate(now, new TimeDuration(-7, TimeUnit.Days));
 
-        await new AnalysisJob({ filepath: LOGS_FILEPATH, since: lastWeek }).execute();
+        await new AnalysisJob({ filepath: LOGS_FILEPATH, since: new Release(1, 8, 1) }).execute();
     }
 
     // Clean logs (remove incomplete sessions)
diff --git a/src/models/Release.ts b/src/models/Release.ts
@@ -39,8 +39,19 @@ class Release implements Comparable {
         this.patch = patch;
     }
 
+    public static fromString(version: string) {
+        const parts = version.split('.');
+        const [major, minor, patch] = parts.map(v => Number(v));
+
+        if (parts.length !== 3 || [major, minor, patch].some(v => Number.isNaN(v))) {
+            throw new Error(`Invalid release format: ${version}`);
+        }
+
+        return new Release(major, minor, patch);
+    }
+
     public toString() {
-        return `v${this.major}.${this.minor}.${this.patch}`;
+        return `${this.major}.${this.minor}.${this.patch}`;
     }
 
     public toJSON() {
diff --git a/src/models/graphs/EventPrevalenceOnWorkdaysGraph.ts b/src/models/graphs/EventPrevalenceOnWorkdaysGraph.ts
@@ -73,9 +73,21 @@ class EventPrevalenceOnWorkdaysGraph extends Graph<SessionHistory> {
         // Sum of error prevalences inside any given bucket should be 100%
         getRange(mergedBuckets.length).forEach((bucketIndex: number) => {
             const sumOfPrevalencesInBucket = uniqueErrors.reduce((prevTotal, error, errorIndex) => {
-                return prevTotal + prevalences[errorIndex].data[bucketIndex].y;
+                const errorPrevalenceInBucket = prevalences[errorIndex].data[bucketIndex].y;
+
+                // Prevalence will be 'NaN' if there was no occurrences in bucket
+                if (Number.isNaN(errorPrevalenceInBucket)) {
+                    return prevTotal;
+                }
+                
+                return prevTotal + errorPrevalenceInBucket;
             }, 0);
 
+            // Ignore buckets with no occurrences
+            if (sumOfPrevalencesInBucket === 0) {
+                return;
+            }
+
             assert(equals(sumOfPrevalencesInBucket, 100) === true);
         });
 
diff --git a/src/models/jobs/AnalysisJob.ts b/src/models/jobs/AnalysisJob.ts
@@ -11,15 +11,16 @@ import SessionLengthGraph from '../graphs/SessionLengthGraph';
 import SessionHistory from '../sessions/SessionHistory';
 import SessionHistoryBuilder from '../sessions/SessionHistoryBuilder';
 import Job from './Job';
+import Release from '../Release';
 
 interface Args {
     filepath: string,
-    since?: Date,
+    since?: Date | Release,
 }
 
 class AnalysisJob extends Job {
     protected filepath: string;
-    protected since?: Date;
+    protected since?: Date | Release;
 
     public constructor(args: Args = { filepath: LOGS_FILEPATH }) {
         super();
diff --git a/src/models/sessions/SessionHistoryBuilder.ts b/src/models/sessions/SessionHistoryBuilder.ts
@@ -9,7 +9,6 @@ import { getRange } from '../../utils/math';
 import SessionBucket from '../buckets/SessionBucket';
 import Release from '../Release';
 import { RELEASE_ZERO } from '../../constants';
-import { MINIMUM_RELEASE } from '../../config';
 
 const TEXTS = {
     SessionStart: '[START]',
@@ -19,8 +18,6 @@ const TEXTS = {
 class SessionHistoryBuilder {
     private static instance: SessionHistoryBuilder;
     
-    private minimumRelease: Release = MINIMUM_RELEASE;
-
     private constructor () {
 
     }
@@ -44,15 +41,6 @@ class SessionHistoryBuilder {
         // Read logs in chronological order
         logs.forEach(log => {
 
-            // Parse app version
-            const [major, minor, patch] = log.version.split('.').map(Number);
-            release = new Release(major, minor, patch);
-
-            // Should builder ignore log?
-            if (release.smallerThan(this.minimumRelease)) {
-                return;
-            }
-
             // Session started
             if (log.msg.includes(TEXTS.SessionStart)) {
                 session = IncompleteSession.create();
diff --git a/src/models/sessions/SessionHistoryExporter.ts b/src/models/sessions/SessionHistoryExporter.ts
@@ -12,7 +12,7 @@ const logToText = (log: Log) => {
         `"time":"${log.time}",` +
         `"pid":${log.pid},` +
         `"hostname":"${log.hostname}",` +
-        `"version":"${log.version}",` +
+        `"version":"${log.version.toString()}",` +
         (log.err ? `"err":"${log.err}",` : '') +
         `"msg":"${log.msg}"` +
         `}`
diff --git a/src/types.ts b/src/types.ts
@@ -1,3 +1,5 @@
+import Release from "./models/Release";
+
 export enum Environment {
     Development = 'development',
     Test = 'test',
@@ -73,7 +75,7 @@ export type CountsDict = Record<string, number>;
 export type Log = {
     line: number,
     time: string,
-    version: string,
+    version: Release,
     level: number,
     pid: number,
     hostname: string,
diff --git a/src/utils/parsing.ts b/src/utils/parsing.ts
@@ -1,21 +1,39 @@
 import { NEW_LINE_REGEXP } from '../constants';
 import logger from '../logger';
+import Release from '../models/Release';
 import { Log } from '../types';
 import { readFile } from './file';
 
 
 
-const isTextLog = (line: string) => line.startsWith('{') && line.endsWith('}');
-const parseTextLog = (line: string, i: number): Log => ({ line: i + 1, ...JSON.parse(line) });
-const hasLogMessage = (log: Log) => !!log && !!log.msg;
+const isTextLog = (line: string) => {
+    return line.startsWith('{') && line.endsWith('}');
+}
 
+const hasLogMessage = (log: Log) => {
+    return !!log && !!log.msg;
+}
 
+const parseTextLog = (line: string, i: number): Log => {
+    const { version, ...log } = JSON.parse(line);
+    return {
+        ...log,
+        line: i + 1,
+        version: Release.fromString(version),
+    };
+}
 
-export const parseLogs = async (filepath: string, since?: Date) => {
+
+
+export const parseLogs = async (filepath: string, since?: Date | Release) => {
     logger.info(`Reading logs from: ${filepath}`);
 
     if (since) {
-        logger.info(`Keeping logs newer than: ${since}`);
+        if (since instanceof Date) {
+            logger.info(`Keeping logs newer than: ${since}`);
+        } else {
+            logger.info(`Keeping logs with release version higher or equal to: ${since.toString()}`);
+        }
     }
 
     const file = await readFile(filepath);
@@ -25,9 +43,13 @@ export const parseLogs = async (filepath: string, since?: Date) => {
         .filter(isTextLog)
         .map(parseTextLog)
         .filter((log: Log) => {
-            if (!since) return true;
-            
-            return new Date(log.time) >= since;
+            if (since instanceof Date) {
+                return new Date(log.time) >= since;
+            }
+            if (since instanceof Release) {
+                return log.version.greaterThanOrEquals(since);
+            }
+            return true;
         })
         .filter(hasLogMessage); // Every log should have a message
 

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "berlin-lea-performance-monitor",`
`3`		`- "version": "1.8.1",`
	`3`	`+ "version": "1.9.0",`
`4`	`4`	`"author": "David Leclerc",`
`5`	`5`	`"main": "./src/index.ts",`
`6`	`6`	`"scripts": {`