Skip to content
This repository was archived by the owner on Aug 12, 2023. It is now read-only.

Commit f62efd7

Browse files
cboviscbovis
authored
Migrate trader metrics to Elasticsearch (#341)
* Migrate checkTraderExists util to elasticsearch * Migrate trader metrics to elasticsearch * Refactor getTraderMetrics module
1 parent f045de8 commit f62efd7

3 files changed

Lines changed: 111 additions & 114 deletions

File tree

‎src/app/routes/v1/metrics.js‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ const createRouter = () => {
8181
router.get(
8282
'/trader',
8383
validatePeriod('period'),
84+
validateGranularity({ period: 'period', granularity: 'granularity' }),
8485
async ({ request, response }, next) => {
8586
const { address } = request.query;
8687

@@ -99,8 +100,11 @@ const createRouter = () => {
99100

100101
const period = request.query.period || TIME_PERIOD.MONTH;
101102

102-
const { dateFrom, dateTo } = getDatesForTimePeriod(period);
103-
const granularity = determineGranularityForTimePeriod(period);
103+
const granularity =
104+
request.query.granularity === undefined
105+
? determineGranularityForTimePeriod(period)
106+
: request.query.granularity;
107+
const { dateFrom, dateTo } = getDatesForMetrics(period, granularity);
104108
const metrics = await getTraderMetrics(
105109
address,
106110
dateFrom,

‎src/metrics/get-trader-metrics.js‎

Lines changed: 99 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -1,125 +1,115 @@
1-
const moment = require('moment');
1+
const _ = require('lodash');
22

3-
const { GRANULARITY } = require('../constants');
4-
const AddressMetric = require('../model/address-metric');
3+
const elasticsearch = require('../util/elasticsearch');
54

6-
const getTraderMetrics = async (address, dateFrom, dateTo, granularity) => {
7-
const dayFrom = moment
8-
.utc(dateFrom)
9-
.startOf('day')
10-
.toDate();
11-
const dayTo = moment
12-
.utc(dateTo)
13-
.endOf('day')
14-
.toDate();
15-
const hourFrom = moment
16-
.utc(dateFrom)
17-
.startOf('hour')
18-
.toDate();
19-
const hourTo = moment
20-
.utc(dateTo)
21-
.endOf('hour')
22-
.toDate();
23-
24-
const pipeline =
25-
granularity === GRANULARITY.DAY
26-
? [
27-
{
28-
$match: { date: { $gte: dayFrom, $lte: dayTo }, address },
5+
const aggregateMetrics = async (
6+
type,
7+
address,
8+
dateFrom,
9+
dateTo,
10+
granularity,
11+
) => {
12+
const results = await elasticsearch.getClient().search({
13+
index: `${type}_metrics_hourly`,
14+
body: {
15+
aggs: {
16+
metrics: {
17+
date_histogram: {
18+
field: 'date',
19+
calendar_interval: granularity,
2920
},
30-
{
31-
$project: {
32-
_id: 0,
33-
date: 1,
34-
fillCount: {
35-
maker: '$fillCount.maker',
36-
taker: '$fillCount.taker',
37-
total: { $ifNull: ['$fillCount.total', '$fillCount'] },
38-
},
39-
fillVolume: {
40-
maker: '$fillVolume.maker',
41-
taker: '$fillVolume.taker',
42-
total: { $ifNull: ['$fillVolume.total', '$fillVolume'] },
43-
},
21+
aggs: {
22+
fillCount: {
23+
sum: { field: 'fillCount' },
4424
},
45-
},
46-
{ $sort: { date: 1 } },
47-
]
48-
: [
49-
{
50-
$match: { date: { $gte: dayFrom, $lte: dayTo }, address },
51-
},
52-
{
53-
$unwind: {
54-
path: '$hours',
25+
fillVolume: {
26+
sum: { field: 'fillVolume' },
5527
},
56-
},
57-
{
58-
$project: {
59-
hour: '$hours.date',
60-
fillCountMaker: '$hours.fillCount.maker',
61-
fillCountTaker: '$hours.fillCount.taker',
62-
fillCountTotal: {
63-
$ifNull: ['$hours.fillCount.total', '$hours.fillCount'],
64-
},
65-
fillVolumeMaker: '$hours.fillVolume.maker',
66-
fillVolumeTaker: '$hours.fillVolume.taker',
67-
fillVolumeTotal: {
68-
$ifNull: ['$hours.fillVolume.total', '$hours.fillVolume'],
69-
},
28+
tradeCount: {
29+
sum: { field: 'tradeCount' },
7030
},
71-
},
72-
{
73-
$match: { hour: { $gte: hourFrom, $lte: hourTo } },
74-
},
75-
{
76-
$group: {
77-
_id: '$hour',
78-
fillCountMaker: {
79-
$sum: '$fillCountMaker',
80-
},
81-
fillCountTaker: {
82-
$sum: '$fillCountTaker',
83-
},
84-
fillCountTotal: {
85-
$sum: '$fillCountTotal',
86-
},
87-
fillVolumeMaker: {
88-
$sum: '$fillVolumeMaker',
89-
},
90-
fillVolumeTaker: {
91-
$sum: '$fillVolumeTaker',
92-
},
93-
fillVolumeTotal: {
94-
$sum: '$fillVolumeTotal',
95-
},
31+
tradeVolume: {
32+
sum: { field: 'tradeVolume' },
9633
},
9734
},
98-
{
99-
$project: {
100-
date: '$_id',
101-
fillCount: {
102-
maker: '$fillCountMaker',
103-
taker: '$fillCountTaker',
104-
total: '$fillCountTotal',
105-
},
106-
fillVolume: {
107-
maker: '$fillVolumeMaker',
108-
taker: '$fillVolumeTaker',
109-
total: '$fillVolumeTotal',
35+
},
36+
},
37+
size: 0,
38+
query: {
39+
bool: {
40+
filter: [
41+
{
42+
range: {
43+
date: {
44+
gte: dateFrom,
45+
lte: dateTo,
46+
},
11047
},
11148
},
112-
},
113-
{
114-
$sort: {
115-
_id: 1,
49+
{
50+
term: {
51+
[type]: address,
52+
},
11653
},
117-
},
118-
];
54+
],
55+
},
56+
},
57+
},
58+
});
59+
60+
return results.body.aggregations.metrics.buckets.map(x => ({
61+
date: new Date(x.key_as_string),
62+
fillCount: x.fillCount.value,
63+
fillVolume: x.fillVolume.value,
64+
tradeCount: x.tradeCount.value,
65+
tradeVolume: x.tradeVolume.value,
66+
}));
67+
};
68+
69+
const reduceMetrics = (makerMetrics, takerMetrics) => {
70+
const dates = _(makerMetrics)
71+
.concat(takerMetrics)
72+
.map(metric => metric.date.toString())
73+
.uniq()
74+
.value();
11975

120-
const dataPoints = await AddressMetric.aggregate(pipeline);
76+
return dates.map(date => {
77+
const takerMetric = takerMetrics.find(
78+
metric => metric.date.toString() === date,
79+
);
80+
81+
const makerMetric = makerMetrics.find(
82+
metric => metric.date.toString() === date,
83+
);
84+
85+
const getMetricValue = key => {
86+
const makerValue = _.get(makerMetric, key, 0);
87+
const takerValue = _.get(takerMetric, key, 0);
88+
89+
return {
90+
maker: makerValue,
91+
taker: takerValue,
92+
total: makerValue + takerValue,
93+
};
94+
};
95+
96+
return {
97+
date: new Date(date),
98+
fillCount: getMetricValue('fillCount'),
99+
fillVolume: getMetricValue('fillVolume'),
100+
tradeCount: getMetricValue('tradeCount'),
101+
tradeVolume: getMetricValue('tradeVolume'),
102+
};
103+
});
104+
};
105+
106+
const getTraderMetrics = async (address, dateFrom, dateTo, granularity) => {
107+
const [makerMetrics, takerMetrics] = await Promise.all([
108+
aggregateMetrics('maker', address, dateFrom, dateTo, granularity),
109+
aggregateMetrics('taker', address, dateFrom, dateTo, granularity),
110+
]);
121111

122-
return dataPoints;
112+
return reduceMetrics(makerMetrics, takerMetrics);
123113
};
124114

125115
module.exports = getTraderMetrics;

‎src/traders/check-trader-exists.js‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
1-
const AddressMetric = require('../model/address-metric');
1+
const elasticsearch = require('../util/elasticsearch');
22

33
const checkTraderExists = async address => {
4-
const results = await AddressMetric.distinct('address', { address });
4+
const response = await elasticsearch.getClient().count({
5+
index: 'trader_metrics_hourly',
6+
body: { query: { term: { trader: address } } },
7+
});
58

6-
return results.length === 1;
9+
return response.body.count > 0;
710
};
811

912
module.exports = checkTraderExists;

0 commit comments

Comments
 (0)