diff --git a/plugins/data_migration/api/api.js b/plugins/data_migration/api/api.js index b60afc5731e..7c47155c46f 100644 --- a/plugins/data_migration/api/api.js +++ b/plugins/data_migration/api/api.js @@ -773,7 +773,7 @@ function trim_ending_slashes(address) { return true; } - if (!params.qstring.only_export || parseInt(params.qstring.only_export) !== 1) { + if (!params.qstring.only_export || (parseInt(params.qstring.only_export) !== 1 && parseInt(params.qstring.only_export) !== 2)) { params.qstring.only_export = false; if (!params.qstring.server_token || params.qstring.server_token === '') { common.returnMessage(params, 404, 'data-migration.token_missing'); @@ -789,6 +789,9 @@ function trim_ending_slashes(address) { } } else { + if (params.qstring.only_export && parseInt(params.qstring.only_export, 10) === 2) { + params.qstring.only_commands = true; + } params.qstring.only_export = true; params.qstring.server_address = ""; params.qstring.server_token = ""; @@ -811,16 +814,30 @@ function trim_ending_slashes(address) { var data_migrator = new migration_helper(); - - data_migrator.export_data(apps, params, common.db, log).then( - function(result) { - common.returnMessage(params, 200, result); - }, - function(error) { - common.returnMessage(params, 404, error.message); - } - - ); + if (params.qstring.only_commands) { + data_migrator.create_export_commands(apps, params, common.db, log).then( + function(result) { + //convert string to buffer + if (typeof result === "string") { + result = Buffer.from(result, 'utf8'); + } + common.returnRaw(params, 200, result, {'Content-Type': 'text/plain; charset=utf-8', 'Content-disposition': 'attachment; filename=countly-export-commands.log'}); + }, + function(error) { + common.returnMessage(params, 404, error.message); + } + ); + } + else { + data_migrator.export_data(apps, params, common.db, log).then( + function(result) { + common.returnMessage(params, 200, result); + }, + function(error) { + common.returnMessage(params, 404, error.message); + } + ); + } }); return true; diff --git a/plugins/data_migration/api/data_migration_helper.js b/plugins/data_migration/api/data_migration_helper.js index bf8a01b55d0..6ad4caf4930 100644 --- a/plugins/data_migration/api/data_migration_helper.js +++ b/plugins/data_migration/api/data_migration_helper.js @@ -413,7 +413,7 @@ module.exports = function(my_db) { } } //new data - scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', "events_data", '-q', '{ "_id": {"$in":{"$regex":"^' + data.appid + '_.*"}}}}', '--out', data.my_folder]}); + scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', "events_data", '-q', '{ "_id": {"$regex":"^' + data.appid + '_.*"}}', '--out', data.my_folder]}); if (plugins.isPluginEnabled('drill')) { scripts.push({cmd: 'mongodump', args: [...data.dbargs_drill, '--collection', "drill_events", '-q', '{ "a": "' + data.appid + '"}', '--out', data.my_folder]}); } @@ -1145,6 +1145,60 @@ module.exports = function(my_db) { this.update_progress = function(my_exportid, step, status, dif, reason, reset_progress, more_fields) { update_progress(my_exportid, step, status, dif, reason, reset_progress, more_fields); }; + + this.create_export_commands = function(apps, my_params, passed_db, passed_log) { + return new Promise(function(resolve, reject) { + if (passed_db) { + db = passed_db; + } + if (my_params) { + params = my_params; + } + if (passed_log) { + log = passed_log; + } + + apps = apps.sort(); + //clear out duplicates + for (let i = 1; i < apps.length - 1; i++) { + if (apps[i - 1] === apps[i]) { + apps.splice(i, 1); i--; + } + } + + var scriptobj = []; + exportid = crypto.createHash('SHA1').update(JSON.stringify(apps)).digest('hex'); + var my_folder = path.resolve(__dirname, './../export/' + exportid); + var image_folder = path.resolve(my_folder, './countly_app_icons'); + for (let i = 0; i < apps.length; i++) { + let subfolder = path.resolve(my_folder, './' + apps[i]); + scriptobj.push({appid: apps[i], my_folder: subfolder, image_folder: image_folder, additional_files: path.resolve(my_folder, './countly_symbolication_files')}); + } + + + Promise.all(scriptobj.map(create_export_scripts)).then(function(result) { + var lines = []; + if (result && Array.isArray(result)) { + for (var i = 0; i < result.length; i++) { + if (Array.isArray(result[i]) && result[i].length > 0) { + for (let j = 0; j < result[i].length; j++) { + lines.push(result[i][j].cmd + " '" + result[i][j].args.join("' '") + "'"); + } + } + } + } + var data = lines.join("\n"); + //save document in gridfs + resolve(data); + + + }).catch(function(err) { + log.e(err); + reject(Error(err.message)); + }); + }); + + }; this.export_data = function(apps, my_params, passed_db, passed_log) { return new Promise(function(resolve, reject) { if (passed_db) { diff --git a/plugins/data_migration/frontend/public/javascripts/countly.models.js b/plugins/data_migration/frontend/public/javascripts/countly.models.js index b4007920ead..25973372a66 100644 --- a/plugins/data_migration/frontend/public/javascripts/countly.models.js +++ b/plugins/data_migration/frontend/public/javascripts/countly.models.js @@ -210,7 +210,12 @@ data: exportData, success: function(json) { if (callback) { - callback({result: "success", data: json.result}); + if (json.result) { + callback({result: "success", data: json.result}); + } + else { + callback({result: "success", data: json}); + } } }, error: function(xhr, status, error) { diff --git a/plugins/data_migration/frontend/public/javascripts/countly.views.js b/plugins/data_migration/frontend/public/javascripts/countly.views.js index 6fa01196096..526cf7d165c 100644 --- a/plugins/data_migration/frontend/public/javascripts/countly.views.js +++ b/plugins/data_migration/frontend/public/javascripts/countly.views.js @@ -313,10 +313,27 @@ countlyDataMigration.saveExport(requestData, function(res) { if (res.result === "success") { - CountlyHelpers.notify({ - type: 'success', - message: CV.i18n('data-migration.export-started') - }); + if (requestData.only_export === 2) { + var data = res.data; + //pack data and download + var blob = new Blob([data], { type: 'application/x-sh' }); + var url = URL.createObjectURL(blob); + var a = document.createElement('a'); + a.href = url; + a.download = 'export_commands.sh'; + document.body.appendChild(a); + a.click(); + CountlyHelpers.notify({ + type: 'success', + message: CV.i18n('data-migration.download-auto') + }); + } + else { + CountlyHelpers.notify({ + type: 'success', + message: CV.i18n('data-migration.export-started') + }); + } } else { CountlyHelpers.notify({ diff --git a/plugins/data_migration/frontend/public/localization/data_migration.properties b/plugins/data_migration/frontend/public/localization/data_migration.properties index e8d4beb8ded..f50aad25f77 100644 --- a/plugins/data_migration/frontend/public/localization/data_migration.properties +++ b/plugins/data_migration/frontend/public/localization/data_migration.properties @@ -25,6 +25,8 @@ data-migration.export-other-path = Export folder: data-migration.export-additional-files = Export crash symbols data-migration.redirect-traffic = Redirect traffic to new server after migration is completed data-migration.export-completed-unable-to-delete = Export completed. Unable to delete files +data-migration.export-type-get-export-scripts = Get only database export commands +data-migration.download-auto = Your download will start automatically. #import data form data-migration.import-title = Import data data-migration.import-type = Import type diff --git a/plugins/data_migration/frontend/public/templates/drawer-export.html b/plugins/data_migration/frontend/public/templates/drawer-export.html index e41fb9ca95d..312edb12508 100644 --- a/plugins/data_migration/frontend/public/templates/drawer-export.html +++ b/plugins/data_migration/frontend/public/templates/drawer-export.html @@ -69,6 +69,13 @@ + +
+ + {{ i18n('data-migration.export-type-get-export-scripts') }} + +
+ diff --git a/plugins/data_migration/scripts/getExportScripts.js b/plugins/data_migration/scripts/getExportScripts.js new file mode 100644 index 00000000000..cff04cd0fb8 --- /dev/null +++ b/plugins/data_migration/scripts/getExportScripts.js @@ -0,0 +1,321 @@ +//PUT in list of appIDS you want to export. If none put in - ALL apps will be exported. +var apps = []; +var export_crashes = true; //Set to false if you do not want to export crashes. +var filePath = "./myfolder"; //Path where to output files when export script runs. + +var plugins = require('./../../../plugins/pluginManager.js'); +var common = require('../../../api/utils/common.js'); +var crypto = require('crypto'); +var db; + +function getApps(countlyDb, callback) { + var query = {}; + if (apps.length > 0) { + var qq = []; + for (var z = 0; z < apps.length; z++) { + qq.push(countlyDb.ObjectID(apps[z])); + } + query = {"_id": {"$in": qq}}; + } + countlyDb.collection("apps").find(query).toArray(callback); +} + +var generate_events_scripts = function(data) { + return new Promise(function(resolve, reject) { + db.collection("events").find({_id: db.ObjectID(data.appid)}).toArray(function(err, res) { + if (err) { + reject(Error(err)); + return; + } + var scripts = []; + if (res && res.length > 0) { + for (var j = 0; j < res.length; j++) { + if (res[j].list && res[j].list.length > 0) { + for (var z = 0; z < res[j].list.length; z++) { + var eventCollName = "events" + crypto.createHash('sha1').update(res[j].list[z] + data.appid).digest('hex'); + //old data, can be removed once we are sure that we are only using merged events_data collection + scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', eventCollName, '--out', data.my_folder]}); + + if (plugins.isPluginEnabled('drill')) { + eventCollName = "drill_events" + crypto.createHash('sha1').update(res[j].list[z] + data.appid).digest('hex'); + scripts.push({cmd: 'mongodump', args: [...data.dbargs_drill, '--collection', eventCollName, '--out', data.my_folder]}); + } + } + } + } + //new data + scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', "events_data", '-q', '{ "_id": {"$regex":"^' + data.appid + '_.*"}}', '--out', data.my_folder]}); + if (plugins.isPluginEnabled('drill')) { + scripts.push({cmd: 'mongodump', args: [...data.dbargs_drill, '--collection', "drill_events", '-q', '{ "a": "' + data.appid + '"}', '--out', data.my_folder]}); + } + } + resolve(scripts); + } + ); + }); +}; + +var generate_credentials_scripts = function(data) { + return new Promise(function(resolve, reject) { + db.collection("apps").findOne({_id: db.ObjectID(data.appid)}, function(err, res) { + if (err) { + reject(Error(err)); + return; + } + var cid = []; + if (res && res.plugins && res.plugins.push) { + if (res.plugins.push.a && res.plugins.push.a._id) { + cid.push('{"$oid":"' + res.plugins.push.a._id + '"}'); + } + + if (res.plugins.push.i && res.plugins.push.i._id) { + cid.push('{"$oid":"' + res.plugins.push.i._id + '"}'); + } + } + if (cid.length > 0) { + resolve([{cmd: 'mongodump', args: [...data.dbargs, '--collection', 'credentials', '-q', '{ "_id": {"$in":[' + cid.join(',') + ']}}', '--out', data.my_folder]}]); + } + else { + resolve([]); + } + }); + }); +}; + +var createScriptsForViews = function(data) { + return new Promise(function(resolve/*, reject*/) { + var scripts = []; + var appId = data.appid; + db.collection("views").findOne({'_id': db.ObjectID(appId)}, {}, function(err, viewInfo) { + + var colName = "app_viewdata" + crypto.createHash('sha1').update(appId).digest('hex'); + scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', colName, '--out', data.my_folder]}); + if (viewInfo) { + for (let segKey in viewInfo.segments) { + colName = "app_viewdata" + crypto.createHash('sha1').update(segKey + appId).digest('hex'); + scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', colName, '--out', data.my_folder]}); + } + } + colName = "app_viewdata" + crypto.createHash('sha1').update('platform' + appId).digest('hex'); + scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', colName, '--out', data.my_folder]}); + resolve(scripts); + }); + + }); +}; + +var create_export_scripts = function(data) { + return new Promise(function(resolve, reject) { + var appid = data.appid; + var my_folder = data.my_folder; + + var scripts = []; + var dbargs = []; + var dbargs0 = []; + var countly_db_name = ""; + var db_params = plugins.getDbConnectionParams('countly'); + for (var p in db_params) { + dbargs.push("--" + p); + dbargs.push(db_params[p]); + if (p !== 'db') { + dbargs0.push("--" + p); + dbargs0.push(db_params[p]); + } + else { + countly_db_name = db_params[p]; + } + } + + var dbargs_drill = []; + db_params = plugins.getDbConnectionParams('countly_drill'); + for (var z in db_params) { + dbargs_drill.push("--" + z); + dbargs_drill.push(db_params[z]); + } + + var dbargs_out = []; + db_params = plugins.getDbConnectionParams('countly_out'); + for (var g in db_params) { + dbargs_out.push("--" + g); + dbargs_out.push(db_params[g]); + } + + db.collection("apps").findOne({_id: db.ObjectID(appid)}, function(err, res) { + if (err || !res) { + reject(Error("data-migration.invalid-app-id")); + } + else { + if (!res.redirect_url || res.redirect_url === "") { + scripts.push({cmd: 'mongodump', args: [...dbargs, "--collection", "apps", "-q", '{ "_id": {"$oid":"' + appid + '"}}', "--out", my_folder]}); + } + else { + //remove redirect field and add it after dump. + scripts.push({cmd: 'mongo', args: [countly_db_name, ...dbargs0, "--eval", 'db.apps.update({ "_id": ObjectId("' + appid + '")}, { "$unset": { "redirect_url": 1 } })']}); + scripts.push({cmd: 'mongodump', args: [...dbargs, "--collection", "apps", "-q", '{ "_id": {"$oid":"' + appid + '"}}', "--out", my_folder]}); + scripts.push({cmd: 'mongo', args: [countly_db_name, ...dbargs0, "--eval", 'db.apps.update({ "_id": ObjectId("' + appid + '")}, { $set: { redirect_url: "' + res.redirect_url + '" } })']}); + } + + var appDocs = ['app_users', 'metric_changes', 'app_crashes', 'app_crashgroups', 'app_crashusers', 'app_nxret', 'app_viewdata', 'app_views', 'app_userviews', 'app_viewsmeta', 'blocked_users', 'campaign_users', 'consent_history', 'crashes_jira', 'event_flows', 'timesofday', 'feedback', 'push_', 'apm', "nps", "survey", "completed_surveys"]; + for (let j = 0; j < appDocs.length; j++) { + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', appDocs[j] + appid, '--out', my_folder]}); + } + + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'campaigndata', '-q', '{ "a": "' + appid + '"}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'campaigns', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'crash_share', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'feedback_widgets', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'notes', '-q', '{ "app_id":"' + appid + '"}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'messages', '-q', '{ "apps": {"$oid":"' + appid + '"}}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'cohortdata', '-q', '{ "a": "' + appid + '"}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'cohorts', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'server_stats_data_points', '-q', '{ "a": "' + appid + '"}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'consent_history', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'flow_schemas', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'flow_data', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'times_of_day', '-q', '{ "app_id": "' + appid + '"}', '--out', my_folder]}); + + //concurrent_users + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'concurrent_users_max', '-q', '{"$or":[{ "app_id": "' + appid + '"},{ "_id": {"$in" :["' + appid + '_overall", "' + appid + '_overall_new"]}}]}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'concurrent_users_alerts', '-q', '{ "app": "' + appid + '"}', '--out', my_folder]}); + + + var sameStructures = ["browser", "carriers", "cities", "consents", "crashdata", "density", "device_details", "devices", "langs", "sources", "users", "retention_daily", "retention_weekly", "retention_monthly", "server_stats_data_points"]; + + for (var k = 0; k < sameStructures.length; k++) { + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', sameStructures[k], '-q', '{ "_id": {"$regex": "^' + appid + '_.*" }}', '--out', my_folder]}); + } + if (dbargs_out && dbargs_out.length) { + scripts.push({cmd: 'mongodump', args: [...dbargs_out, '--collection', "ab_testing_experiments" + appid, '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs_out, '--collection', "remoteconfig_parameters" + appid, '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs_out, '--collection', "remoteconfig_conditions" + appid, '--out', my_folder]}); + } + + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'max_online_counts', '-q', '{"_id": {"$oid":"' + appid + '"}}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'top_events', '-q', '{ "app_id": {"$oid":"' + appid + '"}}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'events', '-q', '{ "_id": {"$oid":"' + appid + '"}}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'views', '-q', '{ "_id": {"$oid":"' + appid + '"}}', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'funnels', '-q', '{ "app_id": "' + appid + '" }', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'calculated_metrics', '-q', '{ "app": "' + appid + '" }', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'datamanager_transforms', '-q', '{ "app": "' + appid + '" }', '--out', my_folder]}); + + + //event Timeline data: + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'eventTimes' + appid, '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'timelineStatus', '-q', '{ "app_id": "' + appid + '" }', '--out', my_folder]}); + + //internal events + for (let j = 0; j < plugins.internalEvents.length; j++) { + let eventCollName = "events" + crypto.createHash('sha1').update(plugins.internalEvents[j] + appid).digest('hex'); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', eventCollName, '--out', my_folder]}); + } + + if (plugins.isPluginEnabled('drill')) { + //export drill + var drill_events = plugins.internalDrillEvents; + + for (let j = 0; j < drill_events.length; j++) { + let eventCollName = "drill_events" + crypto.createHash('sha1').update(drill_events[j] + appid).digest('hex'); + scripts.push({cmd: 'mongodump', args: [...dbargs_drill, '--collection', eventCollName, '--out', my_folder]}); + } + + scripts.push({cmd: 'mongodump', args: [...dbargs_drill, '--collection', 'drill_bookmarks', '-q', '{ "app_id": "' + appid + '" }', '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs_drill, '--collection', 'drill_meta' + appid, '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs_drill, '--collection', 'drill_meta', '-q', '{ "_id": {"$regex": "^' + appid + '_.*" }}', '--out', my_folder]}); + } + //export symbolication files + if (data.aditional_files) { + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'app_crashsymbols' + appid, '--out', my_folder]}); + scripts.push({cmd: 'mongodump', args: [...dbargs, '--collection', 'symbolication_jobs', '-q', '{ "app_id": "' + appid + '" }', '--out', my_folder]}); + } + + //events sctipts + generate_events_scripts({appid: appid, my_folder: my_folder, dbargs: dbargs, dbargs_drill: dbargs_drill}) + .then( + function(result) { + if (result && Array.isArray(result)) { + scripts = scripts.concat(result); + } + + return generate_credentials_scripts({appid: appid, my_folder: my_folder, dbargs: dbargs, dbargs_drill: dbargs_drill}); + }) + .then( + function(result) { + if (result && Array.isArray(result)) { + scripts = scripts.concat(result); + } + + return createScriptsForViews({appid: appid, my_folder: my_folder, dbargs: dbargs, dbargs_drill: dbargs_drill}); + }) + .then( + function(result) { + if (result && Array.isArray(result)) { + scripts = scripts.concat(result); + } + return resolve(scripts); + }, + function(error) { + reject(Error(error.message)); + } + ).catch(err1 => { + reject(err1); + }); + } + }); + }); +}; + +Promise.all([plugins.dbConnection("countly"), plugins.dbConnection("countly_drill")]).then(function([countlyDb, drillDb]) { + common.drillDb = drillDb; + common.db = countlyDb; + db = countlyDb; + plugins.loadConfigs(countlyDb, function() { + getApps(countlyDb, async function(err, apps) { + if (err) { + console.log(err); + console.log("exiting"); + countlyDb.close(); + drillDb.close(); + return; + } + apps = apps || []; + + if (apps.length === 0) { + console.log("0 apps found"); + console.log("exiting"); + countlyDb.close(); + drillDb.close(); + return; + } + else { + try { + // Process apps sequentially with native promises + const results = []; + for (const app of apps) { + const result = await create_export_scripts({ + appid: app._id + "", + my_folder: filePath, + aditional_files: export_crashes + }); + results.push(result); + } + + for (var k = 0; k < results.length; k++) { + console.log("#Scripts for app " + apps[k]._id + ":"); + for (var j = 0; j < results[k].length; j++) { + console.log(results[k][j].cmd + " '" + results[k][j].args.join("' '") + "'"); + } + + } + console.log("# Completed generating export commands."); + countlyDb.close(); + drillDb.close(); + } + catch (error) { + console.log(error); + countlyDb.close(); + drillDb.close(); + } + } + }); + }); +}); \ No newline at end of file