Merge branch 'master' into dependabot/npm_and_yarn/plugins/push/tar-fs-2.1.2

Author: ar2rsawseen

.gitignore

@@ -44,4 +44,5 @@ log/
 log/supervisord/
 plugins/plugins.json.*
 .sdk
-dump
+dump
+dist/

CHANGELOG.md

@@ -1,4 +1,26 @@
-## Version 25.03.x
+## Version 25.03.4
+Features:
+- Add ability to allow multiple CORS per app for web apps
+- Add app id and name as view segment for self-tracking
+- [dashboards] Added the option to set a refresh rate for dashboards, allowing data to update more frequently for selected dashboards
+
+Enterprise Features:
+- [license] Update locking conditions for expired license and over limit usage
+- [license] Enable force locking with remote config
+- [license] Update dashboard lock with redirection to home page for non admin user
+
+Fixes:
+- [core] Allow downloading data also from other databases in dbviewer
+- [crashes] Fix unescaped SDK logs
+- [crash_symbolication] Symbolication server api end point test fix
+- [star-rating] Added missing columns to Rating Widgets table edit
+- [ui] Fix alignment of drawers title and close icon
+- [push] Fixed push notifications title and content text and variables combination
+- [reports] Correctly match event for email report if event key contains '.'
+
+Enterprise Fixes:
+- [cohorts] Fixed issue with combining multiple cohorts
+- [drill] Do not recheck old collections on app_user data deletion if querying from old collections is disabled
 
 Dependencies:
 - Bump body-parser from 1.20.3 to 2.2.0

api/jobs/topEvents.js

@@ -138,14 +138,19 @@ class TopEventsJob extends job.Job {
      * The errors of all functions will be caught here.
      */
     async getAllApps() {
+        log.d("Fetching all apps");
         try {
             const getAllApps = await new Promise((res, rej) => common.db.collection("apps").find({}, { _id: 1, timezone: 1 }).toArray((err, apps) => err ? rej(err) : res(apps)));
-            await Promise.all(getAllApps.map((app) => this.getAppEvents(app)));
+            for (var z = 0; z < getAllApps.length; z++) {
+                //Calculating for each app serially.
+                await this.getAppEvents(getAllApps[z]);
+            }
         }
         catch (error) {
             log.e("TopEvents Job has a error: ", error);
             throw error;
         }
+        log.d("Finished processing");
     }
 
     /**
@@ -178,6 +183,7 @@ class TopEventsJob extends job.Job {
      * @param {Object} app - saveAppEvents object
      */
     async getAppEvents(app) {
+        log.d(app._id + ": Fetching app events");
         const getEvents = await new Promise((res, rej) => common.db.collection("events").findOne({ _id: app._id }, (errorEvents, result) => errorEvents ? rej(errorEvents) : res(result)));
         if (getEvents && 'list' in getEvents) {
             const eventMap = this.eventsFilter(getEvents.list);
@@ -199,6 +205,7 @@ class TopEventsJob extends job.Job {
                     let totalDuration = 0;
                     let prevTotalDuration = 0;
                     for (const event of eventMap) {
+                        log.d("    getting event data for event: " + event + " (" + period + ")");
                         const collectionNameEvents = this.eventsCollentions({ event, id: app._id });
                         await this.getEventsCount({ collectionNameEvents, ob, data, event });
                         totalCount += data[event].data.count.total;
@@ -208,11 +215,16 @@ class TopEventsJob extends job.Job {
                         totalDuration += data[event].data.duration.total;
                         prevTotalDuration += data[event].data.duration["prev-total"];
                     }
+                    log.d("    getting session count (" + period + ")");
                     await this.getSessionCount({ ob, sessionData, usersData, usersCollectionName });
+                    log.d("    saving data (" + period + ")");
                     await this.saveAppEvents({ app, data, sessionData, usersData, period, totalCount, prevTotalCount, totalSum, prevTotalSum, totalDuration, prevTotalDuration });
                 }
             }
         }
+        else {
+            log.d("    No events found for app");
+        }
 
     }
 

api/lib/countly.common.js

@@ -3,8 +3,15 @@
 * @module "api/lib/countly.common"
 */
 
+/**
+ * @typedef {import('moment-timezone').Moment} MomentTimezone
+ */
+
 /** @lends module:api/lib/countly.common */
 var countlyCommon = {},
+    /**
+     * Reference to momentjs
+     */
     moment = require('moment-timezone'),
     underscore = require('underscore');
 
@@ -16,57 +23,11 @@ var _period = "hour",
 // Private Methods
 
 /**
-* Returns array with unique ticks for period
-* @param {moment} startTimestamp - start of period
-* @param {moment} endTimestamp - end of period
-* @returns {array} unique array ticks for period
-**/
-/*function getTicksBetween(startTimestamp, endTimestamp) {
-    var dayIt = startTimestamp.clone(),
-        ticks = [];
-
-    while (dayIt < endTimestamp) {
-        let daysLeft = Math.round(moment.duration(endTimestamp - dayIt).asDays());
-        if (daysLeft >= dayIt.daysInMonth() && dayIt.date() === 1) {
-            ticks.push(dayIt.format("YYYY.M"));
-            dayIt.add(1 + dayIt.daysInMonth() - dayIt.date(), "days");
-        }
-        else if (daysLeft >= (7 - dayIt.day()) && dayIt.day() === 1) {
-            ticks.push(dayIt.format("YYYY.[w]W"));
-            dayIt.add(8 - dayIt.day(), "days");
-        }
-        else {
-            ticks.push(dayIt.format("YYYY.M.D"));
-            dayIt.add(1, "day");
-        }
-    }
-
-    return ticks;
-}*/
-
-/**
-* Returns array with more generalized unique ticks for period
-* @param {moment} startTimestamp - start of period
-* @param {moment} endTimestamp - end of period
-* @returns {array} unique array ticks for period
-**/
-/*function getTicksCheckBetween(startTimestamp, endTimestamp) {
-    var dayIt = startTimestamp.clone(),
-        ticks = [];
-    while (dayIt < endTimestamp) {
-        let daysLeft = Math.round(moment.duration(endTimestamp - dayIt).asDays());
-        if (daysLeft >= (dayIt.daysInMonth() * 0.5 - dayIt.date())) {
-            ticks.push(dayIt.format("YYYY.M"));
-            dayIt.add(1 + dayIt.daysInMonth() - dayIt.date(), "days");
-        }
-        else {
-            ticks.push(dayIt.format("YYYY.[w]W"));
-            dayIt.add(8 - dayIt.day(), "days");
-        }
-    }
-    return ticks;
-}*/
-
+ * Calculates unique values from a hierarchical map structure
+ * @param {Object} dbObj - Database object containing hierarchical data (years, months, weeks, days)
+ * @param {Object} uniqueMap - Map with hierarchical structure (years, months, weeks, days) used to calculate unique values
+ * @returns {number} - Count of unique items
+ */
 countlyCommon.calculateUniqueFromMap = function(dbObj, uniqueMap) {
     var u = 0;
     for (var year in uniqueMap) {
@@ -105,13 +66,13 @@ countlyCommon.calculateUniqueFromMap = function(dbObj, uniqueMap) {
 };
 
 /** returns unique period check array
-        * @param {array} weeksArray_pd - weeks array
-        * @param {array} weekCounts_pd -  week counts
-        * @param {array} monthsArray_pd - months array
-        * @param {array} monthCounts_pd - months counts
-        * @param {array} periodArr_pd - period array
-        * @returns {array} periods
-        */
+* @param {Array<string>} weeksArray_pd - weeks array
+* @param {Array<string>} weekCounts_pd -  week counts
+* @param {Array<string>} monthsArray_pd - months array
+* @param {Array<string>} monthCounts_pd - months counts
+* @param {Array<string>} periodArr_pd - period array
+* @returns {Array<string>} periods
+*/
 function getUniqArray(weeksArray_pd, weekCounts_pd, monthsArray_pd, monthCounts_pd, periodArr_pd) {
 
     if (_period === "month" || _period === "day" || _period === "yesterday" || _period === "hour") {
@@ -230,12 +191,12 @@ function getUniqArray(weeksArray_pd, weekCounts_pd, monthsArray_pd, monthCounts_
     return uniquePeriods;
 }
 /** returns unique period check array
-        * @param {array} weeksArray_pd - weeks array
-        * @param {array} weekCounts_pd -  week counts
-        * @param {array} monthsArray_pd - months array
-        * @param {array} monthCounts_pd - months counts
-        * @returns {array} periods
-        */
+* @param {Array<string>} weeksArray_pd - weeks array
+* @param {Array<string>} weekCounts_pd -  week counts
+* @param {Array<string>} monthsArray_pd - months array
+* @param {Array<string>} monthCounts_pd - months counts
+* @returns {Array<string>} periods
+*/
 function getUniqCheckArray(weeksArray_pd, weekCounts_pd, monthsArray_pd, monthCounts_pd) {
 
     if (_period === "month" || _period === "day" || _period === "yesterday" || _period === "hour") {
@@ -306,9 +267,9 @@ function getUniqCheckArray(weeksArray_pd, weekCounts_pd, monthsArray_pd, monthCo
 }
 
 /** Function to clone object
-        * @param {object} obj - object to clone
-        * @returns {object} cloned object
-        */
+* @param {object} obj - object to clone
+* @returns {object|undefined|string|Array<string>|Date} cloned object
+*/
 function clone(obj) {
     if (null === obj || "object" !== typeof obj) {
         return obj;
@@ -342,8 +303,8 @@ function clone(obj) {
 
 /**
 * Returns number for timestamp making sure it is 13 digits
-* @param {integer}ts -  number we need to se as timestamp
-* @returns {integer} timestamp in ms
+* @param {number} ts - number we need to se as timestamp
+* @returns {number} timestamp in ms
 **/
 function fixTimestampToMilliseconds(ts) {
     if ((ts + "").length > 13) {
@@ -355,7 +316,7 @@ function fixTimestampToMilliseconds(ts) {
 
 /**
 * Returns a period object used by all time related data calculation functions
-* @param {moment} prmPeriod period to be calculated (optional)
+* @param {string|any} prmPeriod period to be calculated (optional) todo:figure this type out
 * @param {string} bucket  - daily or monthly. If bucket is set, period will be modified to fit full months or days
 * @returns {timeObject} time object
 **/

api/parts/data/batcher.js

@@ -422,7 +422,7 @@ class ReadBatcher {
      *  @param {string} id - id of cache
      *  @param {string} query - query for the document
      *  @param {string} projection - which fields to return
-     *  @param {bool} multi - true if multiple documents
+     *  @param {boolean} multi - true if multiple documents
      *  @returns {Promise} promise
      */
     getData(collection, id, query, projection, multi) {
@@ -517,7 +517,7 @@ class ReadBatcher {
      *  @param {string} collection - name of the collection where to update data
      *  @param {object} query - query for the document
      *  @param {object} projection - which fields to return
-     *  @param {bool} multi - true if multiple documents
+     *  @param {boolean} multi - true if multiple documents
      */
     invalidate(collection, query, projection, multi) {
         if (!this.onMaster || cluster.isMaster) {
@@ -539,7 +539,7 @@ class ReadBatcher {
      *  @param {string} collection - name of the collection where to update data
      *  @param {object} query - query for the document
      *  @param {object} projection - which fields to return
-     *  @param {bool} multi - true if multiple documents
+     *  @param {boolean} multi - true if multiple documents
      *  @returns {Promise} promise
      */
     get(collection, query, projection, multi) {
@@ -651,7 +651,7 @@ class ReadBatcher {
      *  @param {string} query - query for the document
      *  @param {string} projection - which fields to return
      *  @param {object} data - data from database
-     *  @param {bool} multi - true if multiple documents
+     *  @param {boolean} multi - true if multiple documents
      */
     cache(collection, id, query, projection, data, multi) {
         if (this.process) {

api/parts/data/usage.js

@@ -398,7 +398,7 @@ usage.returnAllProcessedMetrics = function(params) {
                 continue;
             }
 
-            if (recvMetricValue) {
+            if (recvMetricValue !== undefined && recvMetricValue !== null) {
                 var escapedMetricVal = (recvMetricValue + "").replace(/^\$/, "").replace(/\./g, ":");
                 processedMetrics[tmpMetric.short_code] = escapedMetricVal;
             }
@@ -705,7 +705,7 @@ function processMetrics(user, uniqueLevelsZero, uniqueLevelsMonth, params, done)
                     continue;
                 }
 
-                if (recvMetricValue) {
+                if (recvMetricValue !== undefined && recvMetricValue !== null) {
                     recvMetricValue = (recvMetricValue + "").replace(/^\$/, "").replace(/\./g, ":");
                     postfix = common.crypto.createHash("md5").update(recvMetricValue).digest('base64')[0];
                     metaToFetch[predefinedMetrics[i].db + params.app_id + "_" + dateIds.zero + "_" + postfix] = {
@@ -767,7 +767,7 @@ function processMetrics(user, uniqueLevelsZero, uniqueLevelsMonth, params, done)
                     continue;
                 }
 
-                if (recvMetricValue) {
+                if (recvMetricValue !== undefined && recvMetricValue !== null) {
                     escapedMetricVal = (recvMetricValue + "").replace(/^\$/, "").replace(/\./g, ":");
                     postfix = common.crypto.createHash("md5").update(escapedMetricVal).digest('base64')[0];
 
@@ -1110,6 +1110,15 @@ plugins.register("/sdk/user_properties", async function(ob) {
                 userProps[key] = up[key];
             }
         }
+
+        if (params.qstring.metrics._app_version) {
+            const versionComponents = common.parseAppVersion(params.qstring.metrics._app_version);
+            if (versionComponents.success) {
+                userProps.av_major = versionComponents.major;
+                userProps.av_minor = versionComponents.minor;
+                userProps.av_patch = versionComponents.patch;
+            }
+        }
     }
 
     if (params.qstring.session_duration) {

api/utils/authorizer.js

@@ -15,13 +15,13 @@ const log = require('./log.js')('core:authorizer');
 * @param {object} options - options for the task
 * @param {object} options.db - database connection
 * @param {number} options.ttl - amount of seconds for token to work, 0 works indefinately
-* @param {bool} [options.multi=false] - if true, can be used many times until expired
+* @param {boolean} [options.multi=false] - if true, can be used many times until expired
 * @param {string} options.token - token to store, if not provided, will be generated
 * @param {string} options.owner - id of the user who created this token
 * @param {string} options.app - list of the apps for which token was created
 * @param {string} options.endpoint - regexp of endpoint(any string - is used as substring,to mach exact ^{yourpath}$)
 * @param {string} options.tryReuse - if true - tries to find not expired token with same parameters. If not founds cretes new token. If found - updates token expiration time to new one and returns token.
-* @param {bool} [options.temporary=false] - If logged in with temporary token. Doesn't kill other sessions on logout.
+* @param {boolean} [options.temporary=false] - If logged in with temporary token. Doesn't kill other sessions on logout.
 * @param {function} options.callback - function called when saving was completed or errored, providing error object as first param and token string as second
 */
 authorizer.save = function(options) {