Merge branch 'release.24.10'

# Conflicts:
#	CHANGELOG.md
#	frontend/express/public/javascripts/countly/vue/components/dropdown.js
#	plugins/star-rating/frontend/public/templates/star-consent-link.html
#	ui-tests/cypress/lib/dashboard/feedback/ratings/widgets.js

Author: ar2rsawseen

CHANGELOG.md

@@ -1,12 +1,20 @@
 
 ## Version 25.03.x
 Fixes:
+- [core] Changes for event omit script to validate data in new model and use countly-request.
+- [core] Changes to top events job. Fetching data from aggregated event totals.
+- [crashes] Fix unescaped SDK logs
 - [feedback] Uniformize drawer internal name input texts
+- [feedback] Uniformize feedback widgets status tag
 - [star-rating] Added missing columns to Rating Widgets table edit
+- [star-rating] Allow bulk update of widget status
 - [star-rating] Fix rating score and responses table sorting
 - [ui] Fix alignment of drawers title and close icon
+- [UI] Remove white background from input character amount suffix
 - [heatmaps] Get heatmap data from new drill events collection
 
+Enterprise Fixes:
+ -  [retention] Fixed report loading 
 Fixes:
 - [localization] Fixed grammatical errors
 

api/jobs/topEvents.js

@@ -44,29 +44,49 @@ class TopEventsJob extends job.Job {
     }
 
     /**
-     * async
-     * Get events count.
-     * @param {Object} params - getEventsCount object
-     * @param {String} params.collectionNameEvents - event collection name
-     * @param {Object} params.ob - it contains all necessary info
-     * @param {string} params.event - event name
-     * @param {Object} params.data - dummy event data
-     * @returns {Promise.<boolean>} true.
+     * 
+     * @param {object} params  - params object
+     * @param {object} data  - object where to collect data
+     * @param {boolean} previous  - if fetching for previous period
+     * @returns {Promise} promise
      */
-    async getEventsCount(params) {
-        const { collectionNameEvents, ob, data, event } = params;
+    async fetchEventTotalCounts(params, data, previous) {
+        let collectionName = "all";
+        params.qstring.segmentation = "key";
         return await new Promise((resolve) => {
-            countlyApi.data.fetch.getTimeObjForEvents(collectionNameEvents, ob, (doc) => {
+            countlyApi.data.fetch.getTimeObjForEvents("events_data", params, {'id_prefix': params.app_id + "_" + collectionName + '_'}, function(doc) {
                 countlyEvents.setDb(doc || {});
-                const countProp = countlyEvents.getNumber("c", true);
-                const sumProp = countlyEvents.getNumber("s", true);
-                const durationProp = countlyEvents.getNumber("dur", true);
-                data[event] = {};
-                data[event].data = {
-                    count: countProp,
-                    sum: sumProp,
-                    duration: durationProp
-                };
+
+                var dd = countlyEvents.getSegmentedData(params.qstring.segmentation);
+                for (var z = 0; z < dd.length;z++) {
+                    var key = dd[z]._id;
+                    data[key] = data[key] || {};
+                    data[key].data = data[key].data || {};
+                    data[key].data.count = data[key].data.count || {"total": 0, "prev-total": 0, "change": "NA", "trend": "u"};
+                    if (previous) {
+                        data[key].data.count["prev-total"] = dd[z].c;
+                    }
+                    else {
+                        data[key].data.count.total = dd[z].c;
+                    }
+
+                    data[key].data.sum = data[key].data.sum || {"total": 0, "prev-total": 0, "change": "NA", "trend": "u"};
+                    if (previous) {
+                        data[key].data.sum["prev-total"] = dd[z].s;
+                    }
+                    else {
+                        data[key].data.sum.total = dd[z].s;
+                    }
+
+                    data[key].data.duration = data[key].data.duration || {"total": 0, "prev-total": 0, "change": "NA", "trend": "u"};
+                    if (previous) {
+                        data[key].data.duration["prev-total"] = dd[z].dur;
+                    }
+                    else {
+                        data[key].data.duration.total = dd[z].dur;
+                    }
+                }
+                //data.all = countlyEvents.getSegmentedData(params.qstring.segmentation);
                 resolve(true);
             });
         });
@@ -204,10 +224,20 @@ class TopEventsJob extends job.Job {
                     let prevTotalSum = 0;
                     let totalDuration = 0;
                     let prevTotalDuration = 0;
-                    for (const event of eventMap) {
-                        log.d("    getting event data for event: " + event + " (" + period + ")");
-                        const collectionNameEvents = this.eventsCollentions({ event, id: app._id });
-                        await this.getEventsCount({ collectionNameEvents, ob, data, event });
+
+                    //Fetching totals for this period
+                    await this.fetchEventTotalCounts({ app_id: app._id, appTimezone: app.timezone, qstring: { period: period } }, data, false);
+                    var period2 = countlyCommon.getPeriodObj({appTimezone: app.timezone, qstring: {}}, period);
+                    var newPeriod = [period2.start - (period2.end - period2.start), period2.start];
+                    //Fetching totals for previous period
+                    await this.fetchEventTotalCounts({ app_id: app._id, appTimezone: app.timezone, qstring: { period: newPeriod } }, data, true);
+
+
+                    for (var event in data) {
+                        //Calculating trend
+                        var trend = countlyCommon.getPercentChange(data[event].data.count["prev-total"], data[event].data.count.total);
+                        data[event].data.count.change = trend.percent;
+                        data[event].data.count.trend = trend.trend;
                         totalCount += data[event].data.count.total;
                         prevTotalCount += data[event].data.count["prev-total"];
                         totalSum += data[event].data.sum.total;

bin/scripts/modify-data/omit_events.js

@@ -1,36 +1,30 @@
 /**
  *  Checks aggregated data for all events of all apps and outputs the ones that should need omitting segments. 
+ *  For deletion to work SERVER_URL and API_KEY should be set.
  *  If DRY_RUN is false, will also omit those segmentes
  *  Server: countly
  *  Path: $(countly dir)/bin/scripts/modify-data
  *  Command: nodejs omit_events.js
  */
 
-//API key here with permission to delete events
-var API_KEY = "0d87fb49bd48ddc510306b7b4faf209a";
-
+//API key with global admin rights
+var API_KEY = "";
 //dry run without deleting events
 var DRY_RUN = true;
 var SERVER_URL = "https://yourserver.count.ly";
 
 var requestsToRun = [];
-
+var omitLimit = 100; //Segment value limit
+var failedReqs = 0;
 
 var plugins = require("../../../plugins/pluginManager");
 var crypto = require('crypto');
-var request = require('request');
-var requestOptions = {
-    uri: (process.env.COUNTLY_CONFIG_PROTOCOL || "http") + "://" + (process.env.COUNTLY_CONFIG_HOSTNAME || "localhost") + "/i/events/edit_map",
-    method: 'POST'
-};
+var request = require('countly-request')(plugins.getConfig("security"));
+
 if (!SERVER_URL) {
     SERVER_URL = (process.env.COUNTLY_CONFIG_PROTOCOL || "http") + "://" + (process.env.COUNTLY_CONFIG_HOSTNAME || "localhost");
 }
 plugins.dbConnection().then(async function(db) {
-
-    var date = new Date();
-    var yy = date.getFullYear();
-
     var apps = await db.collection("apps").find().toArray();
     var appCheck = {};
     for (var l = 0; l < apps.length; l++) {
@@ -42,48 +36,78 @@ plugins.dbConnection().then(async function(db) {
         var omitted_segments = {};
         var event_map = {};
         if (events[i] && events[i].list && events[i].list.length) {
+            console.log("Checking app:", events[i]._id);
             for (var j = 0; j < events[i].list.length; j++) {
                 if (events[i].list[j]) {
+                    console.log("    Checking event:", events[i].list[j]);
                     var eventSegmentCounts = {};
-                    var eventMeta = await db.collection("events" + crypto.createHash('sha1').update(events[i].list[j] + events[i]._id).digest('hex')).find({"m": yy + ":0"}).toArray();
-                    for (var k = 0; k < eventMeta.length; k++) {
-                        if (eventMeta[k] && eventMeta[k].meta_v2 && eventMeta[k].meta_v2.segments) {
-                            for (let segment in eventMeta[k].meta_v2.segments) {
-                                if (eventMeta[k].meta_v2[segment]) {
-                                    if (typeof eventSegmentCounts[segment] === "undefined") {
-                                        eventSegmentCounts[segment] = 0;
+                    var hash = crypto.createHash('sha1').update(events[i].list[j] + events[i]._id).digest('hex');
+                    var pipeline = [
+                        {"$match": {"_id": {"$regex": "^" + events[i]._id + "_" + hash + "_no-segment_.*"}, "meta_v2": {"$exists": true}}},
+                        {
+                            "$project": {
+                                "m": "$m",
+                                "meta_v2": {
+                                    "$map": {
+                                        "input": {"$objectToArray": "$meta_v2"},
+                                        "as": "seg",
+                                        "in": {"k": "$$seg.k", "v": {"$size": {"$objectToArray": "$$seg.v"}}}
                                     }
-                                    eventSegmentCounts[segment] += Object.keys(eventMeta[k].meta_v2[segment]).length;
                                 }
                             }
+                        },
+                        {"$unwind": "$meta_v2"},
+                        {
+                            "$group": {
+                                "_id": {"key": "$meta_v2.k", "m": "$m"},
+                                "count": {"$sum": "$meta_v2.v"}
+                            }
+                        }
+                    ];
+                    var eventMeta = await db.collection("events_data").aggregate(pipeline).toArray();
+                    for (var k = 0; k < eventMeta.length; k++) {
+                        if (eventMeta[k]._id.key !== "segments") {
+                            eventSegmentCounts[eventMeta[k]._id.key] = eventSegmentCounts[eventMeta[k]._id.key] || 0;
+                            eventSegmentCounts[eventMeta[k]._id.key] = Math.max(eventMeta[k].count, eventSegmentCounts[eventMeta[k]._id.key]);
                         }
+
                     }
                     var first = true;
                     //console.log("Event:", events[i].list[j], "for app:", events[i]._id);
                     for (let segment in eventSegmentCounts) {
-                        if (eventSegmentCounts[segment] >= 1000) {
+                        if (eventSegmentCounts[segment] >= omitLimit) {
                             if (first) {
                                 if (firstApp) {
-                                    console.log("For app:", appCheck[events[i]._id] || events[i]._id);
                                     firstApp = false;
                                 }
-                                console.log("", "Event:", events[i].list[j]);
                                 first = false;
                             }
-                            console.log("", "", segment, ":", eventSegmentCounts[segment]);
+                            if (events[i] && events[i].omitted_segments && events[i].omitted_segments[events[i].list[j]] && Array.isArray(events[i].omitted_segments[events[i].list[j]])) {
+                                omitted_segments[events[i].list[j]] = events[i].omitted_segments[events[i].list[j]];
+                            }
                             if (!omitted_segments[events[i].list[j]]) {
                                 omitted_segments[events[i].list[j]] = [];
                             }
-                            omitted_segments[events[i].list[j]].push(segment);
+                            if (omitted_segments[events[i].list[j]].indexOf(segment) === -1) {
+                                omitted_segments[events[i].list[j]].push(segment);
+                            }
+
+                            if (events[i] && events[i].map && events[i].map[events[i].list[j]]) {
+                                event_map[events[i].list[j]] = events[i].map[events[i].list[j]];
+                                event_map[events[i].list[j]].omit_list = events[i].map[events[i].list[j]].omit_list || [];
+                            }
                             if (!event_map[events[i].list[j]]) {
                                 event_map[events[i].list[j]] = {key: events[i].list[j], is_visible: true, omit_list: []};
                             }
-                            event_map[events[i].list[j]].omit_list.push(segment);
+                            if (event_map[events[i].list[j]] && event_map[events[i].list[j]].omit_list && event_map[events[i].list[j]].omit_list.indexOf(segment) === -1) {
+                                event_map[events[i].list[j]].omit_list.push(segment);
+                            }
                         }
                     }
                 }
             }
         }
+        console.log("  Current event map:", JSON.stringify(event_map));
         if (Object.keys(omitted_segments).length) {
             if (DRY_RUN) {
                 //as it is dry run  - output request
@@ -96,25 +120,41 @@ plugins.dbConnection().then(async function(db) {
                 requestsToRun.push(SERVER_URL + "/i/events/edit_map?" + props.join("&"));
             }
             else {
-                requestOptions.json = {
-                    omitted_segments: JSON.stringify(omitted_segments),
-                    event_map: JSON.stringify(event_map),
-                    app_id: events[i]._id,
-                    api_key: API_KEY
+                console.log("Omitting segments for app:", events[i]._id, "for events:", JSON.stringify(omitted_segments));
+                const options = {
+                    url: SERVER_URL + "/i/events/edit_map",
+                    uri: SERVER_URL + "/i/events/edit_map",
+                    method: "POST",
+                    json: {
+                        omitted_segments: JSON.stringify(omitted_segments),
+                        event_map: JSON.stringify(event_map),
+                        app_id: events[i]._id + "",
+                        api_key: API_KEY
+                    },
+                    strictSSL: false
                 };
                 await new Promise(function(resolve) {
-                    request(requestOptions, function(error, response, body) {
-                        console.log("request finished", body);
+                    request(SERVER_URL + "/i/events/edit_map", options, function(error) {
+                        if ((error && error.name)) {
+                            failedReqs++;
+                            console.log(JSON.stringify(error.message));
+                            console.log({err: 'There was an error while sending a request.'});
+                        }
+
                         resolve();
                     });
                 });
             }
         }
     }
+    if (failedReqs) {
+        console.log("There were " + failedReqs + " failed requests. Please check your API KEY and url for server");
+    }
     if (requestsToRun.length) {
         for (var z = 0; z < requestsToRun.length; z++) {
             console.log(requestsToRun[z]);
         }
     }
+    console.log("Done");
     db.close();
 });

frontend/express/public/javascripts/countly/vue/components/dropdown.js

@@ -622,7 +622,7 @@
                     else {
                         this.$emit('command', command, instance);
                     }
-                    this.$refs.dropdown.handleClose();
+                    this.$refs?.dropdown?.handleClose();
                 }
             },
             toggleArrowState: function() {

frontend/express/public/javascripts/countly/vue/components/helpers.js

@@ -86,25 +86,63 @@
         }
     }));
 
-    Vue.component("cly-status-tag", countlyBaseComponent.extend({
-        template: '<div class="cly-vue-status-tag" :class="dynamicClasses">\n' +
-                     '<div class="cly-vue-status-tag__blink"></div>\n' +
-                        '{{text}}\n' +
-                  '</div>',
-        mixins: [countlyVue.mixins.i18n],
+    Vue.component('cly-status-tag', countlyBaseComponent.extend({
         props: {
-            text: { required: true, type: String },
-            color: { default: "green", type: String},
-            size: { default: "unset", type: String},
+            color: {
+                default: 'green',
+                type: String
+            },
+
+            loading: {
+                default: false,
+                type: Boolean
+            },
+
+            size: {
+                default: 'unset',
+                type: String
+            },
+
+            text: {
+                required: true,
+                type: String
+            }
         },
+
         computed: {
-            dynamicClasses: function() {
-                if (this.size === "small") {
-                    return ["cly-vue-status-tag--small", "cly-vue-status-tag--" + this.color];
+            dynamicClasses() {
+                const classes = [];
+
+                if (this.size === 'small') {
+                    classes.push('cly-vue-status-tag--small');
+                }
+
+                if (this.loading) {
+                    classes.push('cly-vue-status-tag--gray');
                 }
-                return "cly-vue-status-tag--" + this.color;
+                else {
+                    classes.push(`cly-vue-status-tag--${this.color}`);
+                }
+
+                return classes;
             }
         },
+
+        template: `
+            <div
+                class="cly-vue-status-tag"
+                :class="dynamicClasses"
+            >
+                <div class="cly-vue-status-tag__blink" />
+                <div
+                    v-if="loading"
+                    class="cly-vue-status-tag__skeleton"
+                />
+                <template v-else>
+                    {{ text }}
+                </template>
+            </div>
+        `
     }));
 
     Vue.component("cly-diff-helper", countlyBaseComponent.extend({