Skip to content

Commit 1ace39e

Browse files
CookiezaursCookiezaurs
authored
Merge pull request #6169 from Countly/heatmap-neo-ingestion-master
Update heatmap to use new drill events collection
2 parents 85a6847 + f584fda commit 1ace39e

5 files changed

Lines changed: 384 additions & 141 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ Fixes:
55
- [star-rating] Added missing columns to Rating Widgets table edit
66
- [star-rating] Fix rating score and responses table sorting
77
- [ui] Fix alignment of drawers title and close icon
8+
- [heatmaps] Get heatmap data from new drill events collection
89

910
Fixes:
1011
- [localization] Fixed grammatical errors

‎plugins/views/api/api.js‎

Lines changed: 177 additions & 139 deletions
Original file line numberDiff line numberDiff line change
@@ -1349,178 +1349,216 @@ const escapedViewSegments = { "name": true, "segment": true, "height": true, "wi
13491349
* @param {Object} params - Default parameters object
13501350
* @returns {undefined} Returns nothing
13511351
*/
1352-
function getHeatmap(params) {
1353-
var result = {types: [], data: []};
1352+
async function getHeatmap(params) {
1353+
const result = {types: [], data: [], domains: []};
13541354

1355-
var device = {};
1355+
let device = {};
13561356
try {
13571357
device = JSON.parse(params.qstring.device);
13581358
}
13591359
catch (SyntaxError) {
1360-
console.log('Parse device failed: ', params.qstring.device);
1360+
log.e('Parse device failed: ', params.qstring.device);
13611361
}
13621362

1363-
var actionType = params.qstring.actionType;
1363+
const actionType = params.qstring.actionType;
13641364

13651365
if (!(device.minWidth >= 0) || !(device.maxWidth >= 0)) {
13661366
common.returnMessage(params, 400, 'Bad request parameter: device');
13671367
return false;
13681368
}
1369-
var collectionName = "drill_events" + crypto.createHash('sha1').update("[CLY]_action" + params.qstring.app_id).digest('hex');
1370-
common.drillDb.collection(collectionName).findOne({"_id": "meta_v2"}, {_id: 0, "sg.type": 1, "sg.domain": 1}, function(err1, meta) {
1371-
if (meta && meta.sg && meta.sg.type) {
1372-
result.types = Object.keys(meta.sg.type.values);
1369+
1370+
if (params.qstring.period) {
1371+
//check if period comes from datapicker
1372+
if (params.qstring.period.indexOf(',') !== -1) {
1373+
try {
1374+
params.qstring.period = JSON.parse(params.qstring.period);
1375+
}
1376+
catch (SyntaxError) {
1377+
log.d('Parsing custom period failed!');
1378+
common.returnMessage(params, 400, 'Bad request parameter: period');
1379+
return false;
1380+
}
13731381
}
13741382
else {
1375-
result.types = [];
1383+
switch (params.qstring.period) {
1384+
case 'month':
1385+
case 'day':
1386+
case 'yesterday':
1387+
case 'hour':
1388+
break;
1389+
default:
1390+
if (!/([0-9]+)days/.test(params.qstring.period)) {
1391+
common.returnMessage(params, 400, 'Bad request parameter: period');
1392+
return false;
1393+
}
1394+
break;
1395+
}
13761396
}
1377-
if (meta && meta.sg && meta.sg.domain) {
1378-
result.domains = Object.keys(meta.sg.domain.values).map(function(item) {
1379-
return common.db.decode(item);
1380-
});
1397+
}
1398+
else {
1399+
common.returnMessage(params, 400, 'Missing request parameter: period');
1400+
return false;
1401+
}
1402+
1403+
countlyCommon.setTimezone(params.appTimezone);
1404+
countlyCommon.setPeriod(params.qstring.period);
1405+
1406+
const periodObj = countlyCommon.periodObj;
1407+
const matchQuery = {};
1408+
const now = params.time.now.toDate();
1409+
1410+
//create current period array if it does not exist
1411+
if (!periodObj.currentPeriodArr) {
1412+
periodObj.currentPeriodArr = [];
1413+
1414+
//create a period array that starts from the beginning of the current year until today
1415+
if (params.qstring.period === 'month') {
1416+
for (let i = 0; i < (now.getMonth() + 1); i++) {
1417+
const moment1 = moment();
1418+
const daysInMonth = moment1.month(i).daysInMonth();
1419+
1420+
for (let j = 0; j < daysInMonth; j++) {
1421+
periodObj.currentPeriodArr.push(periodObj.activePeriod + '.' + (i + 1) + '.' + (j + 1));
1422+
1423+
// If current day of current month, just break
1424+
if ((i === now.getMonth()) && (j === (now.getDate() - 1))) {
1425+
break;
1426+
}
1427+
}
1428+
}
13811429
}
1430+
//create a period array that starts from the beginning of the current month until today
1431+
else if (params.qstring.period === 'day') {
1432+
for (let i = 0; i < now.getDate(); i++) {
1433+
periodObj.currentPeriodArr.push(periodObj.activePeriod + '.' + (i + 1));
1434+
}
1435+
}
1436+
//create one day period array
13821437
else {
1383-
result.domains = [];
1438+
periodObj.currentPeriodArr.push(periodObj.activePeriod);
13841439
}
1385-
common.drillDb.collection(collectionName).findOne({"_id": "meta"}, {_id: 0, "sg.type": 1, "sg.domain": 1}, function(err2, meta2) {
1386-
if (meta2 && meta2.sg && meta2.sg.type) {
1387-
common.arrayAddUniq(result.types, meta2.sg.type.values);
1388-
}
1389-
if (meta2 && meta2.sg && meta2.sg.domain) {
1390-
common.arrayAddUniq(result.domains, meta2.sg.domain.values);
1391-
}
1392-
var eventHash = crypto.createHash('sha1').update("[CLY]_action" + params.qstring.app_id).digest('hex');
1393-
var collectionMeta = "drill_meta" + params.qstring.app_id;
1394-
common.drillDb.collection(collectionMeta).findOne({"_id": "meta_" + eventHash}, {_id: 0, "sg.domain": 1}, function(err3, meta_event) {
1395-
if (meta_event && meta_event.sg && meta_event.sg.type) {
1396-
common.arrayAddUniq(result.types, Object.keys(meta_event.sg.type.values));
1397-
}
1398-
if (meta_event && meta_event.sg && meta_event.sg.domain) {
1399-
common.arrayAddUniq(result.domains, Object.keys(meta_event.sg.domain.values));
1400-
}
1440+
}
14011441

1402-
if (params.qstring.period) {
1403-
//check if period comes from datapicker
1404-
if (params.qstring.period.indexOf(",") !== -1) {
1405-
try {
1406-
params.qstring.period = JSON.parse(params.qstring.period);
1407-
}
1408-
catch (SyntaxError) {
1409-
log.d('Parsing custom period failed!');
1410-
common.returnMessage(params, 400, 'Bad request parameter: period');
1411-
return false;
1412-
}
1413-
}
1414-
else {
1415-
switch (params.qstring.period) {
1416-
case "month":
1417-
case "day":
1418-
case "yesterday":
1419-
case "hour":
1420-
break;
1421-
default:
1422-
if (!/([0-9]+)days/.test(params.qstring.period)) {
1423-
common.returnMessage(params, 400, 'Bad request parameter: period');
1424-
return false;
1425-
}
1426-
break;
1427-
}
1428-
}
1429-
}
1430-
else {
1431-
common.returnMessage(params, 400, 'Missing request parameter: period');
1432-
return false;
1433-
}
1434-
countlyCommon.setTimezone(params.appTimezone);
1435-
countlyCommon.setPeriod(params.qstring.period);
1436-
var periodObj = countlyCommon.periodObj,
1437-
queryObject = {},
1438-
now = params.time.now.toDate();
1442+
//get timestamps of start of days (DD-MM-YYYY-00:00) with respect to apptimezone for both beginning and end of period arrays
1443+
let tmpArr;
1444+
const ts = {};
14391445

1440-
//create current period array if it does not exist
1441-
if (!periodObj.currentPeriodArr) {
1442-
periodObj.currentPeriodArr = [];
1446+
tmpArr = periodObj.currentPeriodArr[0].split('.');
1447+
ts.$gte = moment(new Date(Date.UTC(parseInt(tmpArr[0]), parseInt(tmpArr[1]) - 1, parseInt(tmpArr[2]))));
1448+
if (params.appTimezone) {
1449+
ts.$gte.tz(params.appTimezone);
1450+
}
1451+
ts.$gte = ts.$gte.valueOf() - ts.$gte.utcOffset() * 60000;
14431452

1444-
//create a period array that starts from the beginning of the current year until today
1445-
if (params.qstring.period === "month") {
1446-
for (let i = 0; i < (now.getMonth() + 1); i++) {
1447-
var moment1 = moment();
1448-
var daysInMonth = moment1.month(i).daysInMonth();
1453+
tmpArr = periodObj.currentPeriodArr[periodObj.currentPeriodArr.length - 1].split('.');
1454+
ts.$lt = moment(new Date(Date.UTC(parseInt(tmpArr[0]), parseInt(tmpArr[1]) - 1, parseInt(tmpArr[2])))).add(1, 'days');
1455+
if (params.appTimezone) {
1456+
ts.$lt.tz(params.appTimezone);
1457+
}
1458+
ts.$lt = ts.$lt.valueOf() - ts.$lt.utcOffset() * 60000;
1459+
1460+
matchQuery.ts = ts;
1461+
matchQuery.a = params.qstring.app_id;
1462+
matchQuery.e = '[CLY]_action';
1463+
matchQuery['sg.width'] = {};
1464+
matchQuery['sg.width'].$gt = device.minWidth;
1465+
matchQuery['sg.width'].$lte = device.maxWidth;
1466+
matchQuery['sg.type'] = actionType;
1467+
1468+
const projectionQuery = {
1469+
_id: 0,
1470+
c: 1,
1471+
'sg.type': 1,
1472+
'sg.width': 1,
1473+
'sg.height': 1
1474+
};
1475+
1476+
if (actionType === 'scroll') {
1477+
projectionQuery['sg.y'] = 1;
1478+
matchQuery['sg.view'] = params.qstring.view;
1479+
}
1480+
else {
1481+
projectionQuery['sg.x'] = 1;
1482+
projectionQuery['sg.y'] = 1;
1483+
matchQuery['up.lv'] = params.qstring.view;
1484+
}
14491485

1450-
for (var j = 0; j < daysInMonth; j++) {
1451-
periodObj.currentPeriodArr.push(periodObj.activePeriod + "." + (i + 1) + "." + (j + 1));
1486+
if (params.qstring.segment) {
1487+
matchQuery['sg.segment'] = params.qstring.segment;
1488+
}
14521489

1453-
// If current day of current month, just break
1454-
if ((i === now.getMonth()) && (j === (now.getDate() - 1))) {
1455-
break;
1456-
}
1457-
}
1458-
}
1459-
}
1460-
//create a period array that starts from the beginning of the current month until today
1461-
else if (params.qstring.period === "day") {
1462-
for (let i = 0; i < now.getDate(); i++) {
1463-
periodObj.currentPeriodArr.push(periodObj.activePeriod + "." + (i + 1));
1464-
}
1465-
}
1466-
//create one day period array
1467-
else {
1468-
periodObj.currentPeriodArr.push(periodObj.activePeriod);
1469-
}
1470-
}
1490+
const aggregateQuery = [
1491+
{ $match: matchQuery },
1492+
];
1493+
1494+
const use_union_with = plugins.getConfig('drill', params.app_id && params.app && params.app.plugins, true).use_union_with;
1495+
1496+
if (use_union_with) {
1497+
const oldCollectionName = 'drill_events' + crypto.createHash('sha1').update('[CLY]_action' + params.qstring.app_id).digest('hex');
1498+
const eventHash = crypto.createHash('sha1').update('[CLY]_action' + params.qstring.app_id).digest('hex');
1499+
1500+
const metaQuery = [
1501+
{
1502+
$facet: {
1503+
meta: [
1504+
{ $match: { _id: 'meta' } },
1505+
{ $project: { _id: 0, 'sg.type': 1, 'sg.domain': 1 } },
1506+
{ $limit: 1 },
1507+
],
1508+
meta_v2: [
1509+
{ $match: { _id: 'meta_v2' } },
1510+
{ $project: { _id: 0, 'sg.type': 1, 'sg.domain': 1 } },
1511+
{ $limit: 1 },
1512+
],
1513+
},
1514+
},
1515+
];
14711516

1472-
//get timestamps of start of days (DD-MM-YYYY-00:00) with respect to apptimezone for both beginning and end of period arrays
1473-
var tmpArr;
1474-
var ts = {};
1517+
const metaKeys = ['meta', 'meta_v2'];
1518+
const metas = await common.drillDb.collection(oldCollectionName).aggregate(metaQuery).toArray();
1519+
metaKeys.forEach((key) => {
1520+
if (key in metas[0]) {
1521+
const meta = metas[0][key][0];
14751522

1476-
tmpArr = periodObj.currentPeriodArr[0].split(".");
1477-
ts.$gte = moment(new Date(Date.UTC(parseInt(tmpArr[0]), parseInt(tmpArr[1]) - 1, parseInt(tmpArr[2]))));
1478-
if (params.appTimezone) {
1479-
ts.$gte.tz(params.appTimezone);
1523+
if (meta && meta.sg && meta.sg.type && meta.sg.type.values) {
1524+
common.arrayAddUniq(result.types, meta.sg.type.values);
14801525
}
1481-
ts.$gte = ts.$gte.valueOf() - ts.$gte.utcOffset() * 60000;
14821526

1483-
tmpArr = periodObj.currentPeriodArr[periodObj.currentPeriodArr.length - 1].split(".");
1484-
ts.$lt = moment(new Date(Date.UTC(parseInt(tmpArr[0]), parseInt(tmpArr[1]) - 1, parseInt(tmpArr[2])))).add(1, 'days');
1485-
if (params.appTimezone) {
1486-
ts.$lt.tz(params.appTimezone);
1527+
if (meta && meta.sg && meta.sg.domain && meta.sg.domain.values) {
1528+
common.arrayAddUniq(result.domains, meta.sg.domain.values);
14871529
}
1488-
ts.$lt = ts.$lt.valueOf() - ts.$lt.utcOffset() * 60000;
1530+
}
1531+
});
14891532

1490-
queryObject.ts = ts;
1491-
queryObject["sg.width"] = {};
1492-
queryObject["sg.width"].$gt = device.minWidth;
1493-
queryObject["sg.width"].$lte = device.maxWidth;
1494-
queryObject["sg.type"] = actionType;
1533+
const moreMeta = await common.drillDb.collection(`drill_meta${params.qstring.app_id}`).findOne(
1534+
{ _id: `meta_${eventHash}` },
1535+
{ _id: 0, 'sg.domain': 1 },
1536+
);
14951537

1496-
var projections = {
1497-
_id: 0,
1498-
c: 1,
1499-
"sg.type": 1,
1500-
"sg.width": 1,
1501-
"sg.height": 1
1502-
};
1538+
if (moreMeta && moreMeta.sg && moreMeta.sg.domain) {
1539+
common.arrayAddUniq(result.domains, Object.keys(moreMeta.sg.domain.values));
1540+
}
15031541

1504-
if (actionType === "scroll") {
1505-
projections["sg.y"] = 1;
1506-
queryObject["sg.view"] = params.qstring.view;
1507-
}
1508-
else {
1509-
projections["sg.x"] = 1;
1510-
projections["sg.y"] = 1;
1511-
queryObject["up.lv"] = params.qstring.view;
1512-
}
1542+
const matchQueryOld = Object.assign({}, matchQuery);
1543+
delete matchQueryOld.a;
1544+
delete matchQueryOld.e;
15131545

1514-
if (params.qstring.segment) {
1515-
queryObject["sg.segment"] = params.qstring.segment;
1516-
}
1517-
common.drillDb.collection(collectionName).find(queryObject, projections).toArray(function(err, data) {
1518-
result.data = data;
1519-
common.returnOutput(params, result, true, params.token_headers);
1520-
});
1521-
});
1546+
aggregateQuery.push({
1547+
$unionWith: { coll: oldCollectionName, pipeline: [{ $match: matchQueryOld }] },
15221548
});
1523-
});
1549+
}
1550+
1551+
aggregateQuery.push({ $project: projectionQuery });
1552+
1553+
try {
1554+
const data = await common.drillDb.collection('drill_events').aggregate(aggregateQuery, { allowDiskUse: true }).toArray();
1555+
result.data = data;
1556+
common.returnOutput(params, result, true, params.token_headers);
1557+
}
1558+
catch (err) {
1559+
log.e(err);
1560+
common.returnMessage(params, 500, 'Error fetching drill events');
1561+
}
15241562
}
15251563

15261564
plugins.register("/o/actions", function(ob) {

0 commit comments

Comments
 (0)