Merge branch 'newarchitecture' into uj/newarch

Author: kanwarujjaval

api/utils/calculatedDataManager.js

@@ -49,6 +49,36 @@ calculatedDataManager.longtask = async function(options) {
             options5.outputData(null, {"_id": options5.id, "running": true, "data": options5.current_data || {}});
         }
     }
+    /**
+     * Called if there is indication that calculations are in progress
+     * @param {*} options6  - options
+     * @param {*} timeoutObj  - timeout object for returning data       
+     * @param {*} retry  - number of retries left
+     */
+    async function waitForData(options6, timeoutObj, retry) {
+        retry = retry - 1;
+        if (retry <= 0) {
+            return;
+        }
+        else {
+            try {
+                var data = await common.db.collection(collection).findOne({_id: options6.id});
+                if (data && data.data) {
+                    clearTimeout(timeoutObj);
+                    options6.outputData(null, {"_id": options6.id, "lu": data.lu, "data": data.data || {}});
+                    return;
+                }
+                else {
+                    setTimeout(function() {
+                        waitForData(options6, timeoutObj, retry);
+                    }, 2000);
+                }
+            }
+            catch (e) {
+                log.e("Error while getting calculated data", e);
+            }
+        }
+    }
     /** switching to long task
      * @param {object} my_options - options
      * @param {object} my_options.query_data - query data
@@ -64,16 +94,8 @@ calculatedDataManager.longtask = async function(options) {
             await common.db.collection(collection).insertOne({_id: my_options.id, status: "calculating", "lu": new Date()});
         }
         catch (e) {
-            //As could not insert try then getting result
-            var data = await common.db.collection(collection).findOne({_id: options.id});
-            if (data) {
-                my_options.outputData(null, {"data": data.data, "lu": data.lu, "_id": options.id});
-                return;
-            }
-            else {
-                my_options.outputData(e, {"_id": my_options.id, "running": false, "error": true});
-            }
-            clearTimeout(timeout);
+            //As could not insert, it might be calculating already
+            waitForData(my_options, timeout, 10);
             return;
         }
         var start = Date.now().valueOf();
@@ -122,11 +144,17 @@ calculatedDataManager.longtask = async function(options) {
             }
         }
         else if (data.status === "calculating") {
-            if (data.start && (new Date().getTime() - data.start) > 1000 * 60 * 60) {
+            if (data.lu && (new Date().getTime() - new Date(data.lu).getTime()) < 1000 * 60 * 60) {
                 //Return current data if there is any and let it know it is calculating
-                clearTimeout(timeout);
-                options.outputData(null, {"_id": options.id, "running": true, data: data.data || {}});
-                return;
+                if (data.data) {
+                    clearTimeout(timeout);
+                    options.outputData(null, {"_id": options.id, "running": true, data: data.data || {}});
+                    return;
+                }
+                else {
+                    //Do retry each few seconds to check if result is created
+                    waitForData(options, timeout, 10);
+                }
             }
             else {
                 common.db.collection(collection).deleteOne({_id: options.id}, function(ee) {

frontend/express/public/javascripts/countly/vue/components/datatable.js

@@ -75,7 +75,8 @@
             return {
                 controlParams: controlParams,
                 lastSearchQuery: '',
-                firstPage: 1
+                firstPage: 1,
+                lastControlParamsSnapshot: null
             };
         },
 
@@ -328,14 +329,35 @@
             controlParams: {
                 deep: true,
 
-                handler: _.debounce(function() {
-                    // Clear cursor history when search or filters change
-                    if (this.controlParams.searchQuery !== this.lastSearchQuery) {
-                        this.controlParams.cursorHistory = [];
-                        this.lastSearchQuery = this.controlParams.searchQuery;
+                handler: _.debounce(function(newVal) {
+                    // Skip if this is the first run (no snapshot yet)
+                    if (!this.lastControlParamsSnapshot) {
+                        this.lastControlParamsSnapshot = this.getControlParamsSnapshot();
+                        return;
                     }
-                    this.triggerExternalSource();
-                    this.setControlParams();
+
+                    // Check if only internal/derived fields changed (like useCursorPagination)
+                    // These shouldn't trigger a fetch
+                    const meaningfulFields = ['page', 'perPage', 'searchQuery', 'sort', 'cursor', 'selectedDynamicCols'];
+                    const hasMeaningfulChange = meaningfulFields.some(field => {
+                        const newFieldVal = JSON.stringify(newVal[field]);
+                        const oldFieldVal = JSON.stringify(this.lastControlParamsSnapshot[field]);
+                        return newFieldVal !== oldFieldVal;
+                    });
+
+                    // Only trigger fetch if meaningful user-initiated changes occurred
+                    if (hasMeaningfulChange) {
+                        // Clear cursor history when search or filters change
+                        if (this.controlParams.searchQuery !== this.lastSearchQuery) {
+                            this.controlParams.cursorHistory = [];
+                            this.lastSearchQuery = this.controlParams.searchQuery;
+                        }
+                        this.triggerExternalSource();
+                        this.setControlParams();
+                    }
+
+                    // Update snapshot after processing
+                    this.lastControlParamsSnapshot = this.getControlParamsSnapshot();
                 }, 500)
             },
 
@@ -370,6 +392,8 @@
         },
 
         mounted() {
+            // Initialize snapshot on mount to prevent false positives
+            this.lastControlParamsSnapshot = this.getControlParamsSnapshot();
             this.triggerExternalSource();
         },
 
@@ -631,6 +655,19 @@
 
             updateControlParams(newParams) {
                 _.extend(this.controlParams, newParams);
+            },
+
+            getControlParamsSnapshot() {
+                // Create a snapshot of meaningful controlParams fields for comparison
+                // Excludes internal state fields (cursorHistory, useCursorPagination) that get mutated during pagination
+                return {
+                    page: this.controlParams.page,
+                    perPage: this.controlParams.perPage,
+                    searchQuery: this.controlParams.searchQuery,
+                    sort: JSON.parse(JSON.stringify(this.controlParams.sort || [])),
+                    cursor: this.controlParams.cursor,
+                    selectedDynamicCols: JSON.parse(JSON.stringify(this.controlParams.selectedDynamicCols || []))
+                };
             }
         }
     };

plugins/data-manager/frontend/public/localization/data-manager.properties

@@ -209,6 +209,7 @@ data-manager.transformation-method.subheading = Select transformation method tha
 data-manager.incoming = Incoming
 data-manager.existing = Existing
 data-manager.both = Both
+data-manager.clickhouse-incoming-only-message = When using ClickHouse, only incoming data transformations are supported.
 data-manager.rename-type = Rename Type
 data-manager.rename-type.tooltip = Rename Type
 data-manager.rename-with-value = Rename with Specific Value