From 9143a2e31569f28486d5475f1127bc738ee4c2e2 Mon Sep 17 00:00:00 2001
From: Cookiezaurs <>
Date: Fri, 30 May 2025 13:15:05 +0300
Subject: [PATCH 1/4] [data_migration] Fixes for export. Added option to get
just export commands via script or UI.
---
plugins/data_migration/api/api.js | 39 ++-
.../api/data_migration_helper.js | 56 ++-
.../public/javascripts/countly.models.js | 7 +-
.../public/javascripts/countly.views.js | 19 +-
.../localization/data_migration.properties | 2 +
.../public/templates/drawer-export.html | 7 +
.../scripts/getExportScripts.js | 319 ++++++++++++++++++
7 files changed, 432 insertions(+), 17 deletions(-)
create mode 100644 plugins/data_migration/scripts/getExportScripts.js
diff --git a/plugins/data_migration/api/api.js b/plugins/data_migration/api/api.js
index b60afc5731e..5f4bf7bf2cf 100644
--- a/plugins/data_migration/api/api.js
+++ b/plugins/data_migration/api/api.js
@@ -773,7 +773,7 @@ function trim_ending_slashes(address) {
return true;
}
- if (!params.qstring.only_export || parseInt(params.qstring.only_export) !== 1) {
+ if (!params.qstring.only_export || (parseInt(params.qstring.only_export) !== 1 && parseInt(params.qstring.only_export) !== 2)) {
params.qstring.only_export = false;
if (!params.qstring.server_token || params.qstring.server_token === '') {
common.returnMessage(params, 404, 'data-migration.token_missing');
@@ -789,6 +789,9 @@ function trim_ending_slashes(address) {
}
}
else {
+ if (params.qstring.only_export && parseInt(params.qstring.only_export) === 2) {
+ params.qstring.only_commands = true;
+ }
params.qstring.only_export = true;
params.qstring.server_address = "";
params.qstring.server_token = "";
@@ -811,16 +814,30 @@ function trim_ending_slashes(address) {
var data_migrator = new migration_helper();
-
- data_migrator.export_data(apps, params, common.db, log).then(
- function(result) {
- common.returnMessage(params, 200, result);
- },
- function(error) {
- common.returnMessage(params, 404, error.message);
- }
-
- );
+ if (params.qstring.only_commands) {
+ data_migrator.create_export_commands(apps, params, common.db, log).then(
+ function(result) {
+ //convert string to buffer
+ if (typeof result === "string") {
+ result = Buffer.from(result, 'utf8');
+ }
+ common.returnRaw(params, 200, result, {'Content-Type': 'plain/text; charset=utf-8', 'Content-disposition': 'attachment; filename=countly-export-commands.log'});
+ },
+ function(error) {
+ common.returnMessage(params, 404, error.message);
+ }
+ );
+ }
+ else {
+ data_migrator.export_data(apps, params, common.db, log).then(
+ function(result) {
+ common.returnMessage(params, 200, result);
+ },
+ function(error) {
+ common.returnMessage(params, 404, error.message);
+ }
+ );
+ }
});
return true;
diff --git a/plugins/data_migration/api/data_migration_helper.js b/plugins/data_migration/api/data_migration_helper.js
index bf8a01b55d0..cb912ef171c 100644
--- a/plugins/data_migration/api/data_migration_helper.js
+++ b/plugins/data_migration/api/data_migration_helper.js
@@ -413,7 +413,7 @@ module.exports = function(my_db) {
}
}
//new data
- scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', "events_data", '-q', '{ "_id": {"$in":{"$regex":"^' + data.appid + '_.*"}}}}', '--out', data.my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', "events_data", '-q', '{ "_id": {"$regex":"^' + data.appid + '_.*"}}', '--out', data.my_folder]});
if (plugins.isPluginEnabled('drill')) {
scripts.push({cmd: 'mongodump', args: [...data.dbargs_drill, '--collection', "drill_events", '-q', '{ "a": "' + data.appid + '"}', '--out', data.my_folder]});
}
@@ -1145,6 +1145,60 @@ module.exports = function(my_db) {
this.update_progress = function(my_exportid, step, status, dif, reason, reset_progress, more_fields) {
update_progress(my_exportid, step, status, dif, reason, reset_progress, more_fields);
};
+
+ this.create_export_commands = function(apps, my_params, passed_db, passed_log) {
+ return new Promise(function(resolve, reject) {
+ if (passed_db) {
+ db = passed_db;
+ }
+ if (my_params) {
+ params = my_params;
+ }
+ if (passed_log) {
+ log = passed_log;
+ }
+
+ apps = apps.sort();
+ //clear out duplicates
+ for (let i = 1; i < apps.length - 1; i++) {
+ if (apps[i - 1] === apps[i]) {
+ apps.splice(i, 1); i--;
+ }
+ }
+
+ var scriptobj = [];
+ exportid = crypto.createHash('SHA1').update(JSON.stringify(apps)).digest('hex');
+ var my_folder = path.resolve(__dirname, './../export/' + exportid);
+ var image_folder = path.resolve(my_folder, './countly_app_icons');
+ for (let i = 0; i < apps.length; i++) {
+ let subfolder = path.resolve(my_folder, './' + apps[i]);
+ scriptobj.push({appid: apps[i], my_folder: subfolder, image_folder: image_folder, aditional_files: path.resolve(my_folder, './countly_symbolication_files')});
+ }
+
+
+ Promise.all(scriptobj.map(create_export_scripts)).then(function(result) {
+ var lines = [];
+ if (result && Array.isArray(result)) {
+ for (var i = 0; i < result.length; i++) {
+ if (Array.isArray(result[i]) && result[i].length > 0) {
+ for (let j = 0; j < result[i].length; j++) {
+ lines.push(result[i][j].cmd + " '" + result[i][j].args.join("' '") + "'");
+ }
+ }
+ }
+ }
+ var data = lines.join("\n");
+ //save document in gridfs
+ resolve(data);
+
+
+ }).catch(function(err) {
+ log.e(err);
+ reject(Error(err.message));
+ });
+ });
+
+ };
this.export_data = function(apps, my_params, passed_db, passed_log) {
return new Promise(function(resolve, reject) {
if (passed_db) {
diff --git a/plugins/data_migration/frontend/public/javascripts/countly.models.js b/plugins/data_migration/frontend/public/javascripts/countly.models.js
index b4007920ead..25973372a66 100644
--- a/plugins/data_migration/frontend/public/javascripts/countly.models.js
+++ b/plugins/data_migration/frontend/public/javascripts/countly.models.js
@@ -210,7 +210,12 @@
data: exportData,
success: function(json) {
if (callback) {
- callback({result: "success", data: json.result});
+ if (json.result) {
+ callback({result: "success", data: json.result});
+ }
+ else {
+ callback({result: "success", data: json});
+ }
}
},
error: function(xhr, status, error) {
diff --git a/plugins/data_migration/frontend/public/javascripts/countly.views.js b/plugins/data_migration/frontend/public/javascripts/countly.views.js
index 6fa01196096..a05fe596597 100644
--- a/plugins/data_migration/frontend/public/javascripts/countly.views.js
+++ b/plugins/data_migration/frontend/public/javascripts/countly.views.js
@@ -313,10 +313,21 @@
countlyDataMigration.saveExport(requestData, function(res) {
if (res.result === "success") {
- CountlyHelpers.notify({
- type: 'success',
- message: CV.i18n('data-migration.export-started')
- });
+ if (requestData.only_export === 2) {
+ var data = res.data;
+ //pack data and download
+ var blob = new Blob([data], { type: 'application/x-sh' });
+ var url = URL.createObjectURL(blob);
+ var a = document.createElement('a');
+ a.href = url;
+ a.download = 'export_commands.sh';
+ document.body.appendChild(a);
+ a.click();
+ CountlyHelpers.notify({
+ type: 'success',
+ message: CV.i18n('data-migration.download-auto ')
+ });
+ }
}
else {
CountlyHelpers.notify({
diff --git a/plugins/data_migration/frontend/public/localization/data_migration.properties b/plugins/data_migration/frontend/public/localization/data_migration.properties
index e8d4beb8ded..f50aad25f77 100644
--- a/plugins/data_migration/frontend/public/localization/data_migration.properties
+++ b/plugins/data_migration/frontend/public/localization/data_migration.properties
@@ -25,6 +25,8 @@ data-migration.export-other-path = Export folder:
data-migration.export-additional-files = Export crash symbols
data-migration.redirect-traffic = Redirect traffic to new server after migration is completed
data-migration.export-completed-unable-to-delete = Export completed. Unable to delete files
+data-migration.export-type-get-export-scripts = Get only database export commands
+data-migration.download-auto = Your download will start automatically.
#import data form
data-migration.import-title = Import data
data-migration.import-type = Import type
diff --git a/plugins/data_migration/frontend/public/templates/drawer-export.html b/plugins/data_migration/frontend/public/templates/drawer-export.html
index e41fb9ca95d..312edb12508 100644
--- a/plugins/data_migration/frontend/public/templates/drawer-export.html
+++ b/plugins/data_migration/frontend/public/templates/drawer-export.html
@@ -69,6 +69,13 @@
+
+
+
+ {{ i18n('data-migration.export-type-get-export-scripts') }}
+
+
+
diff --git a/plugins/data_migration/scripts/getExportScripts.js b/plugins/data_migration/scripts/getExportScripts.js
new file mode 100644
index 00000000000..18e2598f7af
--- /dev/null
+++ b/plugins/data_migration/scripts/getExportScripts.js
@@ -0,0 +1,319 @@
+//PUT in list of appIDS you want to export. If none put in - ALL apps will be exported.
+var apps = [];
+var export_crashes = true; //Set to false if you do not want to export crashes.
+var filePath = "./myfolder"; //Path where to output files when export script runs.
+
+var plugins = require('./../../../plugins/pluginManager.js');
+var common = require('../../../api/utils/common.js');
+var crypto = require('crypto');
+var db;
+
+function getApps(countlyDb, callback) {
+ var query = {};
+ if (apps.length > 0) {
+ var qq = [];
+ for (var z = 0; z < apps.length; z++) {
+ qq.push(countlyDb.ObjectID(apps[z]));
+ }
+ query = {"_id": {"$in": qq}};
+ }
+ countlyDb.collection("apps").find(query).toArray(callback);
+}
+
+var generate_events_scripts = function(data) {
+ return new Promise(function(resolve, reject) {
+ db.collection("events").find({_id: db.ObjectID(data.appid)}).toArray(function(err, res) {
+ if (err) {
+ reject(Error(err));
+ }
+ var scripts = [];
+ if (res && res.length > 0) {
+ for (var j = 0; j < res.length; j++) {
+ if (res[j].list && res[j].list.length > 0) {
+ for (var z = 0; z < res[j].list.length; z++) {
+ var eventCollName = "events" + crypto.createHash('sha1').update(res[j].list[z] + data.appid).digest('hex');
+ //old data, can be removed once we are sure that we are only using merged events_data collection
+ scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', eventCollName, '--out', data.my_folder]});
+
+ if (plugins.isPluginEnabled('drill')) {
+ eventCollName = "drill_events" + crypto.createHash('sha1').update(res[j].list[z] + data.appid).digest('hex');
+ scripts.push({cmd: 'mongodump', args: [...data.dbargs_drill, '--collection', eventCollName, '--out', data.my_folder]});
+ }
+ }
+ }
+ }
+ //new data
+ scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', "events_data", '-q', '{ "_id": {"$regex":"^' + data.appid + '_.*"}}', '--out', data.my_folder]});
+ if (plugins.isPluginEnabled('drill')) {
+ scripts.push({cmd: 'mongodump', args: [...data.dbargs_drill, '--collection', "drill_events", '-q', '{ "a": "' + data.appid + '"}', '--out', data.my_folder]});
+ }
+ }
+ resolve(scripts);
+ }
+ );
+ });
+};
+
+var generate_credentials_scripts = function(data) {
+ return new Promise(function(resolve, reject) {
+ db.collection("apps").findOne({_id: db.ObjectID(data.appid)}, function(err, res) {
+ if (err) {
+ reject(Error(err));
+ }
+ var cid = [];
+ if (res && res.plugins && res.plugins.push) {
+ if (res.plugins.push.a && res.plugins.push.a._id) {
+ cid.push('{"$oid":"' + res.plugins.push.a._id + '"}');
+ }
+
+ if (res.plugins.push.i && res.plugins.push.i._id) {
+ cid.push('{"$oid":"' + res.plugins.push.i._id + '"}');
+ }
+ }
+ if (cid.length > 0) {
+ resolve([{cmd: 'mongodump', args: [...data.dbargs, '--collection', 'credentials', '-q', '{ "_id": {"$in":[' + cid.join(',') + ']}}', '--out', data.my_folder]}]);
+ }
+ else {
+ resolve([]);
+ }
+ });
+ });
+};
+
+var createScriptsForViews = function(data) {
+ return new Promise(function(resolve/*, reject*/) {
+ var scripts = [];
+ var appId = data.appid;
+ db.collection("views").findOne({'_id': db.ObjectID(appId)}, {}, function(err, viewInfo) {
+
+ var colName = "app_viewdata" + crypto.createHash('sha1').update(appId).digest('hex');
+ scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', colName, '--out', data.my_folder]});
+ if (viewInfo) {
+ for (let segKey in viewInfo.segments) {
+ colName = "app_viewdata" + crypto.createHash('sha1').update(segKey + appId).digest('hex');
+ scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', colName, '--out', data.my_folder]});
+ }
+ }
+ colName = "app_viewdata" + crypto.createHash('sha1').update('platform' + appId).digest('hex');
+ scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', colName, '--out', data.my_folder]});
+ resolve(scripts);
+ });
+
+ });
+};
+
+var create_export_scripts = function(data) {
+ return new Promise(function(resolve, reject) {
+ var appid = data.appid;
+ var my_folder = data.my_folder;
+
+ var scripts = [];
+ var dbargs = [];
+ var dbargs0 = [];
+ var countly_db_name = "";
+ var db_params = plugins.getDbConnectionParams('countly');
+ for (var p in db_params) {
+ dbargs.push("--" + p);
+ dbargs.push(db_params[p]);
+ if (p !== 'db') {
+ dbargs0.push("--" + p);
+ dbargs0.push(db_params[p]);
+ }
+ else {
+ countly_db_name = db_params[p];
+ }
+ }
+
+ var dbargs_drill = [];
+ db_params = plugins.getDbConnectionParams('countly_drill');
+ for (var z in db_params) {
+ dbargs_drill.push("--" + z);
+ dbargs_drill.push(db_params[z]);
+ }
+
+ var dbargs_out = [];
+ db_params = plugins.getDbConnectionParams('countly_out');
+ for (var g in db_params) {
+ dbargs_out.push("--" + g);
+ dbargs_out.push(db_params[g]);
+ }
+
+ db.collection("apps").findOne({_id: db.ObjectID(appid)}, function(err, res) {
+ if (err || !res) {
+ reject(Error("data-migration.invalid-app-id"));
+ }
+ else {
+ if (!res.redirect_url || res.redirect_url === "") {
+ scripts.push({cmd: 'mongodump', args: [...dbargs, "--collection", "apps", "-q", '{ "_id": {"$oid":"' + appid + '"}}', "--out", my_folder]});
+ }
+ else {
+ //remove redirect field and add it after dump.
+ scripts.push({cmd: 'mongo', args: [countly_db_name, ...dbargs0, "--eval", 'db.apps.update({ "_id": ObjectId("' + appid + '")}, { "$unset": { "redirect_url": 1 } })']});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, "--collection", "apps", "-q", '{ "_id": {"$oid":"' + appid + '"}}', "--out", my_folder]});
+ scripts.push({cmd: 'mongo', args: [countly_db_name, ...dbargs0, "--eval", 'db.apps.update({ "_id": ObjectId("' + appid + '")}, { $set: { redirect_url: "' + res.redirect_url + '" } })']});
+ }
+
+ var appDocs = ['app_users', 'metric_changes', 'app_crashes', 'app_crashgroups', 'app_crashusers', 'app_nxret', 'app_viewdata', 'app_views', 'app_userviews', 'app_viewsmeta', 'blocked_users', 'campaign_users', 'consent_history', 'crashes_jira', 'event_flows', 'timesofday', 'feedback', 'push_', 'apm', "nps", "survey", "completed_surveys"];
+ for (let j = 0; j < appDocs.length; j++) {
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', appDocs[j] + appid, '--out', my_folder]});
+ }
+
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'campaigndata', '-q', '{ "a": "' + appid + '"}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'campaigns', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'crash_share', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'feedback_widgets', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'notes', '-q', '{ "app_id":"' + appid + '"}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'messages', '-q', '{ "apps": {"$oid":"' + appid + '"}}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'cohortdata', '-q', '{ "a": "' + appid + '"}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'cohorts', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'server_stats_data_points', '-q', '{ "a": "' + appid + '"}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'consent_history', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'flow_schemas', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'flow_data', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'times_of_day', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]});
+
+ //concurrent_users
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'concurrent_users_max', '-q', '{"$or":[{ "app_id": "' + appid + '"},{ "_id": {"$in" :["' + appid + '_overall", "' + appid + '_overall_new"]}}]}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'concurrent_users_alerts', '-q', '{ "app": "' + appid + '"}', '--out', my_folder]});
+
+
+ var sameStructures = ["browser", "carriers", "cities", "consents", "crashdata", "density", "device_details", "devices", "langs", "sources", "users", "retention_daily", "retention_weekly", "retention_monthly", "server_stats_data_points"];
+
+ for (var k = 0; k < sameStructures.length; k++) {
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', sameStructures[k], '-q', '{ "_id": {"$regex": "^' + appid + '_.*" }}', '--out', my_folder]});
+ }
+ if (dbargs_out && dbargs_out.length) {
+ scripts.push({cmd: 'mongodump', args: [...dbargs_out, '--collection', "ab_testing_experiments" + appid, '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs_out, '--collection', "remoteconfig_parameters" + appid, '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs_out, '--collection', "remoteconfig_conditions" + appid, '--out', my_folder]});
+ }
+
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'max_online_counts', '-q', '{"_id": {"$oid":"' + appid + '"}}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'top_events', '-q', '{ "app_id": {"$oid":"' + appid + '"}}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'events', '-q', '{ "_id": {"$oid":"' + appid + '"}}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'views', '-q', '{ "_id": {"$oid":"' + appid + '"}}', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'funnels', '-q', '{ "app_id": "' + appid + '" }', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'calculated_metrics', '-q', '{ "app": "' + appid + '" }', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'datamanager_transforms', '-q', '{ "app": "' + appid + '" }', '--out', my_folder]});
+
+
+ //event Timeline data:
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'eventTimes' + appid, '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'timelineStatus', '-q', '{ "app_id": "' + appid + '" }', '--out', my_folder]});
+
+ //internal events
+ for (let j = 0; j < plugins.internalEvents.length; j++) {
+ let eventCollName = "events" + crypto.createHash('sha1').update(plugins.internalEvents[j] + appid).digest('hex');
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', eventCollName, '--out', my_folder]});
+ }
+
+ if (plugins.isPluginEnabled('drill')) {
+ //export drill
+ var drill_events = plugins.internalDrillEvents;
+
+ for (let j = 0; j < drill_events.length; j++) {
+ let eventCollName = "drill_events" + crypto.createHash('sha1').update(drill_events[j] + appid).digest('hex');
+ scripts.push({cmd: 'mongodump', args: [...dbargs_drill, '--collection', eventCollName, '--out', my_folder]});
+ }
+
+ scripts.push({cmd: 'mongodump', args: [...dbargs_drill, '--collection', 'drill_bookmarks', '-q', '{ "app_id": "' + appid + '" }', '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs_drill, '--collection', 'drill_meta' + appid, '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs_drill, '--collection', 'drill_meta', '-q', '{ "_id": {"$regex": "^' + appid + '_.*" }}', '--out', my_folder]});
+ }
+ //export symbolication files
+ if (data.aditional_files) {
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'app_crashsymbols' + appid, '--out', my_folder]});
+ scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'symbolication_jobs', '-q', '{ "app_id": "' + appid + '" }', '--out', my_folder]});
+ }
+
+ //events sctipts
+ generate_events_scripts({appid: appid, my_folder: my_folder, dbargs: dbargs, dbargs_drill: dbargs_drill}, db)
+ .then(
+ function(result) {
+ if (result && Array.isArray(result)) {
+ scripts = scripts.concat(result);
+ }
+
+ return generate_credentials_scripts({appid: appid, my_folder: my_folder, dbargs: dbargs, dbargs_drill: dbargs_drill});
+ })
+ .then(
+ function(result) {
+ if (result && Array.isArray(result)) {
+ scripts = scripts.concat(result);
+ }
+
+ return createScriptsForViews({appid: appid, my_folder: my_folder, dbargs: dbargs, dbargs_drill: dbargs_drill});
+ })
+ .then(
+ function(result) {
+ if (result && Array.isArray(result)) {
+ scripts = scripts.concat(result);
+ }
+ return resolve(scripts);
+ },
+ function(error) {
+ reject(Error(error.message));
+ }
+ ).catch(err1 => {
+ reject(err1);
+ });
+ }
+ });
+ });
+};
+
+Promise.all([plugins.dbConnection("countly"), plugins.dbConnection("countly_drill")]).then(function([countlyDb, drillDb]) {
+ common.drillDb = drillDb;
+ common.db = countlyDb;
+ db = countlyDb;
+ plugins.loadConfigs(countlyDb, function() {
+ getApps(countlyDb, async function(err, apps) {
+ if (err) {
+ console.log(err);
+ console.log("exiting");
+ countlyDb.close();
+ drillDb.close();
+ return;
+ }
+ apps = apps || [];
+
+ if (apps.length === 0) {
+ console.log("0 apps found");
+ console.log("exiting");
+ countlyDb.close();
+ drillDb.close();
+ return;
+ }
+ else {
+ try {
+ // Process apps sequentially with native promises
+ const results = [];
+ for (const app of apps) {
+ const result = await create_export_scripts({
+ appid: app._id + "",
+ my_folder: filePath,
+ aditional_files: export_crashes
+ });
+ results.push(result);
+ }
+
+ for (var k = 0; k < results.length; k++) {
+ console.log("#Scripts for app " + apps[k]._id + ":");
+ for (var j = 0; j < results[k].length; j++) {
+ console.log(results[k][j].cmd + " '" + results[k][j].args.join("' '") + "'");
+ }
+
+ }
+ console.log("# Completed generating export commands.");
+ countlyDb.close();
+ drillDb.close();
+ }
+ catch (error) {
+ console.log(error);
+ countlyDb.close();
+ drillDb.close();
+ }
+ }
+ });
+ });
+});
\ No newline at end of file
From 2c99068a911e75a7a1f46c164a56e593619b9a1b Mon Sep 17 00:00:00 2001
From: Cookiezaurs <>
Date: Fri, 30 May 2025 13:38:58 +0300
Subject: [PATCH 2/4] deepscan fixes
---
plugins/data_migration/scripts/getExportScripts.js | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/plugins/data_migration/scripts/getExportScripts.js b/plugins/data_migration/scripts/getExportScripts.js
index 18e2598f7af..cff04cd0fb8 100644
--- a/plugins/data_migration/scripts/getExportScripts.js
+++ b/plugins/data_migration/scripts/getExportScripts.js
@@ -25,6 +25,7 @@ var generate_events_scripts = function(data) {
db.collection("events").find({_id: db.ObjectID(data.appid)}).toArray(function(err, res) {
if (err) {
reject(Error(err));
+ return;
}
var scripts = [];
if (res && res.length > 0) {
@@ -59,6 +60,7 @@ var generate_credentials_scripts = function(data) {
db.collection("apps").findOne({_id: db.ObjectID(data.appid)}, function(err, res) {
if (err) {
reject(Error(err));
+ return;
}
var cid = [];
if (res && res.plugins && res.plugins.push) {
@@ -227,7 +229,7 @@ var create_export_scripts = function(data) {
}
//events sctipts
- generate_events_scripts({appid: appid, my_folder: my_folder, dbargs: dbargs, dbargs_drill: dbargs_drill}, db)
+ generate_events_scripts({appid: appid, my_folder: my_folder, dbargs: dbargs, dbargs_drill: dbargs_drill})
.then(
function(result) {
if (result && Array.isArray(result)) {
From edc53588d47575cf407a9e566d4a9cbf502e2d21 Mon Sep 17 00:00:00 2001
From: Anna Sosina
Date: Fri, 30 May 2025 13:44:16 +0300
Subject: [PATCH 3/4] Update
plugins/data_migration/frontend/public/javascripts/countly.views.js
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
---
.../data_migration/frontend/public/javascripts/countly.views.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/plugins/data_migration/frontend/public/javascripts/countly.views.js b/plugins/data_migration/frontend/public/javascripts/countly.views.js
index a05fe596597..71c6418f5e5 100644
--- a/plugins/data_migration/frontend/public/javascripts/countly.views.js
+++ b/plugins/data_migration/frontend/public/javascripts/countly.views.js
@@ -325,7 +325,7 @@
a.click();
CountlyHelpers.notify({
type: 'success',
- message: CV.i18n('data-migration.download-auto ')
+ message: CV.i18n('data-migration.download-auto')
});
}
}
From a4bf0e97e8e474817f31ba3318410adca0e39b1c Mon Sep 17 00:00:00 2001
From: Cookiezaurs <>
Date: Fri, 30 May 2025 14:00:01 +0300
Subject: [PATCH 4/4] fx
---
plugins/data_migration/api/api.js | 4 ++--
plugins/data_migration/api/data_migration_helper.js | 2 +-
.../frontend/public/javascripts/countly.views.js | 6 ++++++
3 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/plugins/data_migration/api/api.js b/plugins/data_migration/api/api.js
index 5f4bf7bf2cf..7c47155c46f 100644
--- a/plugins/data_migration/api/api.js
+++ b/plugins/data_migration/api/api.js
@@ -789,7 +789,7 @@ function trim_ending_slashes(address) {
}
}
else {
- if (params.qstring.only_export && parseInt(params.qstring.only_export) === 2) {
+ if (params.qstring.only_export && parseInt(params.qstring.only_export, 10) === 2) {
params.qstring.only_commands = true;
}
params.qstring.only_export = true;
@@ -821,7 +821,7 @@ function trim_ending_slashes(address) {
if (typeof result === "string") {
result = Buffer.from(result, 'utf8');
}
- common.returnRaw(params, 200, result, {'Content-Type': 'plain/text; charset=utf-8', 'Content-disposition': 'attachment; filename=countly-export-commands.log'});
+ common.returnRaw(params, 200, result, {'Content-Type': 'text/plain; charset=utf-8', 'Content-disposition': 'attachment; filename=countly-export-commands.log'});
},
function(error) {
common.returnMessage(params, 404, error.message);
diff --git a/plugins/data_migration/api/data_migration_helper.js b/plugins/data_migration/api/data_migration_helper.js
index cb912ef171c..6ad4caf4930 100644
--- a/plugins/data_migration/api/data_migration_helper.js
+++ b/plugins/data_migration/api/data_migration_helper.js
@@ -1172,7 +1172,7 @@ module.exports = function(my_db) {
var image_folder = path.resolve(my_folder, './countly_app_icons');
for (let i = 0; i < apps.length; i++) {
let subfolder = path.resolve(my_folder, './' + apps[i]);
- scriptobj.push({appid: apps[i], my_folder: subfolder, image_folder: image_folder, aditional_files: path.resolve(my_folder, './countly_symbolication_files')});
+ scriptobj.push({appid: apps[i], my_folder: subfolder, image_folder: image_folder, additional_files: path.resolve(my_folder, './countly_symbolication_files')});
}
diff --git a/plugins/data_migration/frontend/public/javascripts/countly.views.js b/plugins/data_migration/frontend/public/javascripts/countly.views.js
index a05fe596597..b01933fa38c 100644
--- a/plugins/data_migration/frontend/public/javascripts/countly.views.js
+++ b/plugins/data_migration/frontend/public/javascripts/countly.views.js
@@ -328,6 +328,12 @@
message: CV.i18n('data-migration.download-auto ')
});
}
+ else {
+ CountlyHelpers.notify({
+ type: 'success',
+ message: CV.i18n('data-migration.export-started')
+ });
+ }
}
else {
CountlyHelpers.notify({