Skip to content

Commit 2f18ed7

Browse files
CookiezaursCookiezaurs
authored
[data_migration] Fixes for export. Added option to get just export commands via script or UI.
[data_migration] Fixes for export. Added option to get just export commands via script or UI.
2 parents 0e24cc6 + e41e7d4 commit 2f18ed7

7 files changed

Lines changed: 440 additions & 17 deletions

File tree

‎plugins/data_migration/api/api.js‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -773,7 +773,7 @@ function trim_ending_slashes(address) {
773773
return true;
774774
}
775775

776-
if (!params.qstring.only_export || parseInt(params.qstring.only_export) !== 1) {
776+
if (!params.qstring.only_export || (parseInt(params.qstring.only_export) !== 1 && parseInt(params.qstring.only_export) !== 2)) {
777777
params.qstring.only_export = false;
778778
if (!params.qstring.server_token || params.qstring.server_token === '') {
779779
common.returnMessage(params, 404, 'data-migration.token_missing');
@@ -789,6 +789,9 @@ function trim_ending_slashes(address) {
789789
}
790790
}
791791
else {
792+
if (params.qstring.only_export && parseInt(params.qstring.only_export, 10) === 2) {
793+
params.qstring.only_commands = true;
794+
}
792795
params.qstring.only_export = true;
793796
params.qstring.server_address = "";
794797
params.qstring.server_token = "";
@@ -811,16 +814,30 @@ function trim_ending_slashes(address) {
811814

812815

813816
var data_migrator = new migration_helper();
814-
815-
data_migrator.export_data(apps, params, common.db, log).then(
816-
function(result) {
817-
common.returnMessage(params, 200, result);
818-
},
819-
function(error) {
820-
common.returnMessage(params, 404, error.message);
821-
}
822-
823-
);
817+
if (params.qstring.only_commands) {
818+
data_migrator.create_export_commands(apps, params, common.db, log).then(
819+
function(result) {
820+
//convert string to buffer
821+
if (typeof result === "string") {
822+
result = Buffer.from(result, 'utf8');
823+
}
824+
common.returnRaw(params, 200, result, {'Content-Type': 'text/plain; charset=utf-8', 'Content-disposition': 'attachment; filename=countly-export-commands.log'});
825+
},
826+
function(error) {
827+
common.returnMessage(params, 404, error.message);
828+
}
829+
);
830+
}
831+
else {
832+
data_migrator.export_data(apps, params, common.db, log).then(
833+
function(result) {
834+
common.returnMessage(params, 200, result);
835+
},
836+
function(error) {
837+
common.returnMessage(params, 404, error.message);
838+
}
839+
);
840+
}
824841

825842
});
826843
return true;

‎plugins/data_migration/api/data_migration_helper.js‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -413,7 +413,7 @@ module.exports = function(my_db) {
413413
}
414414
}
415415
//new data
416-
scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', "events_data", '-q', '{ "_id": {"$in":{"$regex":"^' + data.appid + '_.*"}}}}', '--out', data.my_folder]});
416+
scripts.push({cmd: 'mongodump', args: [...data.dbargs, '--collection', "events_data", '-q', '{ "_id": {"$regex":"^' + data.appid + '_.*"}}', '--out', data.my_folder]});
417417
if (plugins.isPluginEnabled('drill')) {
418418
scripts.push({cmd: 'mongodump', args: [...data.dbargs_drill, '--collection', "drill_events", '-q', '{ "a": "' + data.appid + '"}', '--out', data.my_folder]});
419419
}
@@ -1145,6 +1145,60 @@ module.exports = function(my_db) {
11451145
this.update_progress = function(my_exportid, step, status, dif, reason, reset_progress, more_fields) {
11461146
update_progress(my_exportid, step, status, dif, reason, reset_progress, more_fields);
11471147
};
1148+
1149+
this.create_export_commands = function(apps, my_params, passed_db, passed_log) {
1150+
return new Promise(function(resolve, reject) {
1151+
if (passed_db) {
1152+
db = passed_db;
1153+
}
1154+
if (my_params) {
1155+
params = my_params;
1156+
}
1157+
if (passed_log) {
1158+
log = passed_log;
1159+
}
1160+
1161+
apps = apps.sort();
1162+
//clear out duplicates
1163+
for (let i = 1; i < apps.length - 1; i++) {
1164+
if (apps[i - 1] === apps[i]) {
1165+
apps.splice(i, 1); i--;
1166+
}
1167+
}
1168+
1169+
var scriptobj = [];
1170+
exportid = crypto.createHash('SHA1').update(JSON.stringify(apps)).digest('hex');
1171+
var my_folder = path.resolve(__dirname, './../export/' + exportid);
1172+
var image_folder = path.resolve(my_folder, './countly_app_icons');
1173+
for (let i = 0; i < apps.length; i++) {
1174+
let subfolder = path.resolve(my_folder, './' + apps[i]);
1175+
scriptobj.push({appid: apps[i], my_folder: subfolder, image_folder: image_folder, additional_files: path.resolve(my_folder, './countly_symbolication_files')});
1176+
}
1177+
1178+
1179+
Promise.all(scriptobj.map(create_export_scripts)).then(function(result) {
1180+
var lines = [];
1181+
if (result && Array.isArray(result)) {
1182+
for (var i = 0; i < result.length; i++) {
1183+
if (Array.isArray(result[i]) && result[i].length > 0) {
1184+
for (let j = 0; j < result[i].length; j++) {
1185+
lines.push(result[i][j].cmd + " '" + result[i][j].args.join("' '") + "'");
1186+
}
1187+
}
1188+
}
1189+
}
1190+
var data = lines.join("\n");
1191+
//save document in gridfs
1192+
resolve(data);
1193+
1194+
1195+
}).catch(function(err) {
1196+
log.e(err);
1197+
reject(Error(err.message));
1198+
});
1199+
});
1200+
1201+
};
11481202
this.export_data = function(apps, my_params, passed_db, passed_log) {
11491203
return new Promise(function(resolve, reject) {
11501204
if (passed_db) {

‎plugins/data_migration/frontend/public/javascripts/countly.models.js‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,12 @@
210210
data: exportData,
211211
success: function(json) {
212212
if (callback) {
213-
callback({result: "success", data: json.result});
213+
if (json.result) {
214+
callback({result: "success", data: json.result});
215+
}
216+
else {
217+
callback({result: "success", data: json});
218+
}
214219
}
215220
},
216221
error: function(xhr, status, error) {

‎plugins/data_migration/frontend/public/javascripts/countly.views.js‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -313,10 +313,27 @@
313313

314314
countlyDataMigration.saveExport(requestData, function(res) {
315315
if (res.result === "success") {
316-
CountlyHelpers.notify({
317-
type: 'success',
318-
message: CV.i18n('data-migration.export-started')
319-
});
316+
if (requestData.only_export === 2) {
317+
var data = res.data;
318+
//pack data and download
319+
var blob = new Blob([data], { type: 'application/x-sh' });
320+
var url = URL.createObjectURL(blob);
321+
var a = document.createElement('a');
322+
a.href = url;
323+
a.download = 'export_commands.sh';
324+
document.body.appendChild(a);
325+
a.click();
326+
CountlyHelpers.notify({
327+
type: 'success',
328+
message: CV.i18n('data-migration.download-auto')
329+
});
330+
}
331+
else {
332+
CountlyHelpers.notify({
333+
type: 'success',
334+
message: CV.i18n('data-migration.export-started')
335+
});
336+
}
320337
}
321338
else {
322339
CountlyHelpers.notify({

‎plugins/data_migration/frontend/public/localization/data_migration.properties‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ data-migration.export-other-path = Export folder:
2525
data-migration.export-additional-files = Export crash symbols
2626
data-migration.redirect-traffic = Redirect traffic to new server after migration is completed
2727
data-migration.export-completed-unable-to-delete = Export completed. Unable to delete files
28+
data-migration.export-type-get-export-scripts = Get only database export commands
29+
data-migration.download-auto = Your download will start automatically.
2830
#import data form
2931
data-migration.import-title = Import data
3032
data-migration.import-type = Import type

‎plugins/data_migration/frontend/public/templates/drawer-export.html‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,13 @@
6969
</span>
7070
</div>
7171
</el-radio>
72+
<el-radio class="data-migration__drawer__export-radio-item bu-is-flex bu-mt-3" :label="2" border>
73+
<div class="data-migration__drawer__radio-label">
74+
<span>
75+
{{ i18n('data-migration.export-type-get-export-scripts') }}
76+
</span>
77+
</div>
78+
</el-radio>
7279
</el-radio-group>
7380
</div>
7481
</div>

0 commit comments

Comments
 (0)