diff --git a/src/constants.js b/src/constants.js index 55c6f6ae1..286939df8 100644 --- a/src/constants.js +++ b/src/constants.js @@ -86,6 +86,7 @@ module.exports = { FETCH_ADDRESS_TYPE: 'fetch-address-type', FETCH_TOKEN_METADATA: 'fetch-token-metadata', FETCH_TRANSACTION: 'fetch-transaction', + INDEX_APP_FILLS: 'index-app-fills', INDEX_FILL: 'index-fill', INDEX_FILL_PROTOCOL_FEE: 'index-fill-protocol-fee', INDEX_FILL_TRADERS: 'index-fill-traders', diff --git a/src/consumers/index-app-fills/index.js b/src/consumers/index-app-fills/index.js new file mode 100644 index 000000000..8add9a95e --- /dev/null +++ b/src/consumers/index-app-fills/index.js @@ -0,0 +1,117 @@ +const _ = require('lodash'); + +const { JOB, QUEUE, FILL_ATTRIBUTION_TYPE } = require('../../constants'); +const { publishJob } = require('../../queues'); +const elasticsearch = require('../../util/elasticsearch'); +const getAddressMetadata = require('../../addresses/get-address-metadata'); +const getIndexName = require('../../index/get-index-name'); +const getTransactionByHash = require('../../transactions/get-transaction-by-hash'); + +const indexAppFills = async (job, { logger }) => { + const delayJobProcessing = async () => { + await publishJob(QUEUE.INDEXING, JOB.INDEX_APP_FILLS, job.data, { + delay: 30000, + }); + }; + + const { + attributions, + fillDate, + fillId, + maker, + taker, + tradeCount, + tradeValue, + transactionHash, + } = job.data; + + const takerMetadata = await getAddressMetadata(taker); + + if (takerMetadata === null || takerMetadata.isContract === undefined) { + logger.warn(`taker address type is unknown: ${taker}`); + await delayJobProcessing(); + + return; + } + + let transaction; + + if (takerMetadata.isContract) { + transaction = await getTransactionByHash(transactionHash); + + if (transaction === null) { + logger.warn(`transaction has not been fetched: ${transactionHash}`); + await delayJobProcessing(); + + return; + } + } + + const traders = [maker, takerMetadata.isContract ? transaction.from : taker]; + + const appFills = _(attributions) + .filter( + attribution => + attribution.type === FILL_ATTRIBUTION_TYPE.CONSUMER || + attribution.type === FILL_ATTRIBUTION_TYPE.RELAYER, + ) + .map(attribution => attribution.entityId) + .uniq() + .map(appId => { + const isRelayer = attributions.some( + a => a.entityId === appId && a.type === FILL_ATTRIBUTION_TYPE.RELAYER, + ); + + return { + appId, + date: fillDate, + fillId, + relayedTradeCount: isRelayer ? tradeCount : 0, + relayedTradeValue: isRelayer || !tradeValue ? tradeValue : 0, + totalTradeCount: tradeCount, + totalTradeValue: tradeValue, + traders, + }; + }) + .value(); + + if (appFills.length === 0) { + logger.info(`skipped app_fills indexing for fill: ${fillId}`); + + return; + } + + const requestBody = appFills + .map(appFill => + [ + JSON.stringify({ + index: { + _id: `${appFill.fillId}_${appFill.appId}`, + }, + }), + JSON.stringify(appFill), + ].join('\n'), + ) + .join('\n'); + + const result = await elasticsearch + .getClient() + .bulk({ body: `${requestBody}\n`, index: getIndexName('app_fills') }); + + if (result.body.errors === true) { + const errorMessage = _.get( + result, + 'body.items[0].index.error.reason', + `Indexing failed`, + ); + throw new Error(errorMessage); + } + + logger.info(`populated app_fills index for fill: ${fillId}`); +}; + +module.exports = { + fn: indexAppFills, + jobName: JOB.INDEX_APP_FILLS, + queueName: QUEUE.INDEXING, +}; diff --git a/src/consumers/index-app-fills/index.test.js b/src/consumers/index-app-fills/index.test.js new file mode 100644 index 000000000..870d467b8 --- /dev/null +++ b/src/consumers/index-app-fills/index.test.js @@ -0,0 +1,659 @@ +const { Client } = require('@elastic/elasticsearch'); +const Mock = require('@elastic/elasticsearch-mock'); + +const { getModel } = require('../../model'); +const { + mockLogger, + resetDb, + setupDb, + tearDownDb, +} = require('../../test-utils'); +const { publishJob } = require('../../queues'); +const elasticsearch = require('../../util/elasticsearch'); +const consumer = require('.'); + +jest.mock('../../queues'); +jest.mock('../../util/elasticsearch'); + +const elasticsearchMock = new Mock(); +const mockOptions = { + logger: mockLogger, +}; + +beforeAll(async () => { + const client = new Client({ + node: 'http://localhost:9200', + Connection: elasticsearchMock.getConnection(), + }); + + elasticsearch.getClient.mockReturnValue(client); + await setupDb(); +}, 30000); + +afterEach(async () => { + jest.clearAllMocks(); + await resetDb(); + elasticsearchMock.clearAll(); +}, 30000); + +afterAll(async () => { + await tearDownDb(); +}, 30000); + +describe('consumers/index-app-fills', () => { + it('should consume indexing queue', () => { + expect(consumer.queueName).toBe('indexing'); + }); + + it('should consume index-app-fills jobs', () => { + expect(consumer.jobName).toBe('index-app-fills'); + }); + + it('should delay job processing when taker type unknown', async () => { + const job = { + data: { + attributions: [], + fillDate: new Date('2020-08-02T07:47:28Z'), + fillId: '5f267c7b545e125452c56e14', + maker: '0x903153f55770b7668a497180f7fa93471545ffe2', + taker: '0xd0f8715fda0c1b564c2087315fb55804eaf1fae9', + tradeCount: 1, + tradeValue: 500, + transactionHash: + '0x165a1b8f4fbcf089e48435c2efef12dab5223c560f067f932c8efc3d7c6d74eb', + }, + }; + + await consumer.fn(job, mockOptions); + + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(mockLogger.warn).toHaveBeenCalledWith( + 'taker address type is unknown: 0xd0f8715fda0c1b564c2087315fb55804eaf1fae9', + ); + + expect(publishJob).toHaveBeenCalledTimes(1); + expect(publishJob).toHaveBeenCalledWith( + 'indexing', + 'index-app-fills', + job.data, + { delay: 30000 }, + ); + }); + + it('should delay job processing when taker is contract and associated transaction not fetched', async () => { + const job = { + data: { + attributions: [], + fillDate: new Date('2020-08-02T07:47:28Z'), + fillId: '5f267c7b545e125452c56e14', + maker: '0x903153f55770b7668a497180f7fa93471545ffe2', + taker: '0xd0f8715fda0c1b564c2087315fb55804eaf1fae9', + tradeCount: 1, + tradeValue: 500, + transactionHash: + '0x165a1b8f4fbcf089e48435c2efef12dab5223c560f067f932c8efc3d7c6d74eb', + }, + }; + + const AddressMetadata = getModel('AddressMetadata'); + + await AddressMetadata.create({ + address: '0xd0f8715fda0c1b564c2087315fb55804eaf1fae9', + isContract: true, + }); + + await consumer.fn(job, mockOptions); + + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(mockLogger.warn).toHaveBeenCalledWith( + 'transaction has not been fetched: 0x165a1b8f4fbcf089e48435c2efef12dab5223c560f067f932c8efc3d7c6d74eb', + ); + + expect(publishJob).toHaveBeenCalledTimes(1); + expect(publishJob).toHaveBeenCalledWith( + 'indexing', + 'index-app-fills', + job.data, + { delay: 30000 }, + ); + }); + + it('should index a single document when relayer and consumer are the same', async () => { + const job = { + data: { + attributions: [ + { + entityId: '052b4862-2142-4532-bdc0-416814b0a5fe', + type: 0, + }, + { + entityId: '052b4862-2142-4532-bdc0-416814b0a5fe', + type: 1, + }, + { + entityId: 'bdb40272-89f6-4972-b88c-b9baf0ef5410', + type: 2, + }, + ], + fillDate: new Date('2021-04-15T00:54:44.000Z'), + fillId: '60778fa4d2090c3c2ae307f7', + maker: '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + taker: '0xd965952823153e5cbc611be87e8322cfc329f056', + tradeCount: 1, + tradeValue: 12137.05, + transactionHash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + }, + }; + + let indexingBody; + + elasticsearchMock.add( + { + method: 'POST', + path: '/app_fills/_bulk', + }, + ({ body }) => { + indexingBody = body; + + return { status: 'ok' }; + }, + ); + + const AddressMetadata = getModel('AddressMetadata'); + + await AddressMetadata.create({ + address: '0xd965952823153e5cbc611be87e8322cfc329f056', + isContract: false, + }); + + await consumer.fn(job, mockOptions); + + expect(indexingBody).toEqual([ + { + index: { + _id: '60778fa4d2090c3c2ae307f7_052b4862-2142-4532-bdc0-416814b0a5fe', + }, + }, + { + appId: '052b4862-2142-4532-bdc0-416814b0a5fe', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 1, + relayedTradeValue: 12137.05, + totalTradeCount: 1, + totalTradeValue: 12137.05, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xd965952823153e5cbc611be87e8322cfc329f056', + ], + }, + ]); + }); + + it('should index two documents when relayer and consumer are different', async () => { + const job = { + data: { + attributions: [ + { + entityId: '052b4862-2142-4532-bdc0-416814b0a5fe', + type: 0, + }, + { + entityId: '5067df8b-f9cd-4a34-aee1-38d607100145', + type: 1, + }, + { + entityId: 'bdb40272-89f6-4972-b88c-b9baf0ef5410', + type: 2, + }, + ], + fillDate: new Date('2021-04-15T00:54:44.000Z'), + fillId: '60778fa4d2090c3c2ae307f7', + maker: '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + taker: '0xd965952823153e5cbc611be87e8322cfc329f056', + tradeCount: 1, + tradeValue: 12137.05, + transactionHash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + }, + }; + + let indexingBody; + + elasticsearchMock.add( + { + method: 'POST', + path: '/app_fills/_bulk', + }, + ({ body }) => { + indexingBody = body; + + return { status: 'ok' }; + }, + ); + + const AddressMetadata = getModel('AddressMetadata'); + + await AddressMetadata.create({ + address: '0xd965952823153e5cbc611be87e8322cfc329f056', + isContract: false, + }); + + await consumer.fn(job, mockOptions); + + expect(indexingBody).toEqual([ + { + index: { + _id: '60778fa4d2090c3c2ae307f7_052b4862-2142-4532-bdc0-416814b0a5fe', + }, + }, + { + appId: '052b4862-2142-4532-bdc0-416814b0a5fe', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 1, + relayedTradeValue: 12137.05, + totalTradeCount: 1, + totalTradeValue: 12137.05, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xd965952823153e5cbc611be87e8322cfc329f056', + ], + }, + { + index: { + _id: '60778fa4d2090c3c2ae307f7_5067df8b-f9cd-4a34-aee1-38d607100145', + }, + }, + { + appId: '5067df8b-f9cd-4a34-aee1-38d607100145', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 0, + relayedTradeValue: 0, + totalTradeCount: 1, + totalTradeValue: 12137.05, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xd965952823153e5cbc611be87e8322cfc329f056', + ], + }, + ]); + }); + + it('should skip indexing when fill does not have any app attributions', async () => { + const job = { + data: { + attributions: [ + { + entityId: 'bdb40272-89f6-4972-b88c-b9baf0ef5410', + type: 2, + }, + ], + fillDate: new Date('2021-04-15T00:54:44.000Z'), + fillId: '60778fa4d2090c3c2ae307f7', + maker: '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + taker: '0xd965952823153e5cbc611be87e8322cfc329f056', + tradeCount: 1, + tradeValue: 12137.05, + transactionHash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + }, + }; + + let indexingBody; + + elasticsearchMock.add( + { + method: 'POST', + path: '/app_fills/_bulk', + }, + ({ body }) => { + indexingBody = body; + + return { status: 'ok' }; + }, + ); + + const AddressMetadata = getModel('AddressMetadata'); + + await AddressMetadata.create({ + address: '0xd965952823153e5cbc611be87e8322cfc329f056', + isContract: false, + }); + + await consumer.fn(job, mockOptions); + + expect(indexingBody).toBeUndefined(); + expect(mockLogger.info).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + 'skipped app_fills indexing for fill: 60778fa4d2090c3c2ae307f7', + ); + }); + + it('should index single document when fill only has a relayer', async () => { + const job = { + data: { + attributions: [ + { + entityId: '052b4862-2142-4532-bdc0-416814b0a5fe', + type: 0, + }, + { + entityId: 'bdb40272-89f6-4972-b88c-b9baf0ef5410', + type: 2, + }, + ], + fillDate: new Date('2021-04-15T00:54:44.000Z'), + fillId: '60778fa4d2090c3c2ae307f7', + maker: '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + taker: '0xd965952823153e5cbc611be87e8322cfc329f056', + tradeCount: 1, + tradeValue: 12137.05, + transactionHash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + }, + }; + + let indexingBody; + + elasticsearchMock.add( + { + method: 'POST', + path: '/app_fills/_bulk', + }, + ({ body }) => { + indexingBody = body; + + return { status: 'ok' }; + }, + ); + + const AddressMetadata = getModel('AddressMetadata'); + + await AddressMetadata.create({ + address: '0xd965952823153e5cbc611be87e8322cfc329f056', + isContract: false, + }); + + await consumer.fn(job, mockOptions); + + expect(indexingBody).toEqual([ + { + index: { + _id: '60778fa4d2090c3c2ae307f7_052b4862-2142-4532-bdc0-416814b0a5fe', + }, + }, + { + appId: '052b4862-2142-4532-bdc0-416814b0a5fe', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 1, + relayedTradeValue: 12137.05, + totalTradeCount: 1, + totalTradeValue: 12137.05, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xd965952823153e5cbc611be87e8322cfc329f056', + ], + }, + ]); + + expect(mockLogger.info).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + 'populated app_fills index for fill: 60778fa4d2090c3c2ae307f7', + ); + }); + + it('should index single document when fill only has a consumer', async () => { + const job = { + data: { + attributions: [ + { + entityId: '5067df8b-f9cd-4a34-aee1-38d607100145', + type: 1, + }, + { + entityId: 'bdb40272-89f6-4972-b88c-b9baf0ef5410', + type: 2, + }, + ], + fillDate: new Date('2021-04-15T00:54:44.000Z'), + fillId: '60778fa4d2090c3c2ae307f7', + maker: '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + taker: '0xd965952823153e5cbc611be87e8322cfc329f056', + tradeCount: 1, + tradeValue: 12137.05, + transactionHash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + }, + }; + + let indexingBody; + + elasticsearchMock.add( + { + method: 'POST', + path: '/app_fills/_bulk', + }, + ({ body }) => { + indexingBody = body; + + return { status: 'ok' }; + }, + ); + + const AddressMetadata = getModel('AddressMetadata'); + + await AddressMetadata.create({ + address: '0xd965952823153e5cbc611be87e8322cfc329f056', + isContract: false, + }); + + await consumer.fn(job, mockOptions); + + expect(indexingBody).toEqual([ + { + index: { + _id: '60778fa4d2090c3c2ae307f7_5067df8b-f9cd-4a34-aee1-38d607100145', + }, + }, + { + appId: '5067df8b-f9cd-4a34-aee1-38d607100145', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 0, + relayedTradeValue: 0, + totalTradeCount: 1, + totalTradeValue: 12137.05, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xd965952823153e5cbc611be87e8322cfc329f056', + ], + }, + ]); + + expect(mockLogger.info).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + 'populated app_fills index for fill: 60778fa4d2090c3c2ae307f7', + ); + }); + + it('should index without trade value if unavailable', async () => { + const job = { + data: { + attributions: [ + { + entityId: '052b4862-2142-4532-bdc0-416814b0a5fe', + type: 0, + }, + { + entityId: '5067df8b-f9cd-4a34-aee1-38d607100145', + type: 1, + }, + { + entityId: 'bdb40272-89f6-4972-b88c-b9baf0ef5410', + type: 2, + }, + ], + fillDate: new Date('2021-04-15T00:54:44.000Z'), + fillId: '60778fa4d2090c3c2ae307f7', + maker: '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + taker: '0xd965952823153e5cbc611be87e8322cfc329f056', + tradeCount: 1, + tradeValue: undefined, + transactionHash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + }, + }; + + let indexingBody; + + elasticsearchMock.add( + { + method: 'POST', + path: '/app_fills/_bulk', + }, + ({ body }) => { + indexingBody = body; + + return { status: 'ok' }; + }, + ); + + const AddressMetadata = getModel('AddressMetadata'); + + await AddressMetadata.create({ + address: '0xd965952823153e5cbc611be87e8322cfc329f056', + isContract: false, + }); + + await consumer.fn(job, mockOptions); + + expect(indexingBody).toEqual([ + { + index: { + _id: '60778fa4d2090c3c2ae307f7_052b4862-2142-4532-bdc0-416814b0a5fe', + }, + }, + { + appId: '052b4862-2142-4532-bdc0-416814b0a5fe', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 1, + totalTradeCount: 1, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xd965952823153e5cbc611be87e8322cfc329f056', + ], + }, + { + index: { + _id: '60778fa4d2090c3c2ae307f7_5067df8b-f9cd-4a34-aee1-38d607100145', + }, + }, + { + appId: '5067df8b-f9cd-4a34-aee1-38d607100145', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 0, + totalTradeCount: 1, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xd965952823153e5cbc611be87e8322cfc329f056', + ], + }, + ]); + + expect(mockLogger.info).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + 'populated app_fills index for fill: 60778fa4d2090c3c2ae307f7', + ); + }); + + it('should index transaction sender as taker when taker address is a contract', async () => { + const job = { + data: { + attributions: [ + { + entityId: '5067df8b-f9cd-4a34-aee1-38d607100145', + type: 1, + }, + ], + fillDate: new Date('2021-04-15T00:54:44.000Z'), + fillId: '60778fa4d2090c3c2ae307f7', + maker: '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + taker: '0xd965952823153e5cbc611be87e8322cfc329f056', + tradeCount: 1, + tradeValue: 12137.05, + transactionHash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + }, + }; + + const AddressMetadata = getModel('AddressMetadata'); + const Transaction = getModel('Transaction'); + + let indexingBody; + + elasticsearchMock.add( + { + method: 'POST', + path: '/app_fills/_bulk', + }, + ({ body }) => { + indexingBody = body; + + return { status: 'ok' }; + }, + ); + + await AddressMetadata.create({ + address: '0xd965952823153e5cbc611be87e8322cfc329f056', + isContract: true, + }); + + await Transaction.create({ + blockHash: + '0xb7a0059c1aff2e84f7d5b75dd29b57325bd6a8dd0469240689cc52df1df1146a', + blockNumber: 10893374, + data: + '0x000d070a11ced27fd19c7a2bc70fd72adb55bb7025014e00000000000000000000d3d2e2692501a5c9ca623199d38826e513033a17000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000001f9840a85d5af5bf1d1762f925bdaddc4201f984000000000000000000000000e33c8e3a0d14a81f0dd7e174830089e82f65fc85000000000000000000000000000000000000000000000003255b68cdf304cdc40000000000000000000000000000000000000000000000031e171c6104e502420000000000000000000000000000000000000000000000101ad5ea23c2190000014e000000000000000000005e8d405cbc564473d85a9a31fbfca76167d6997800000000000000000000000086003b044f70dac0abc80ac8957305b6370893ed00000000000000000000000000000000000000000000001043561a882930000000000000000000000000000000000000000000000000039f985e76e5df87c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005f6a05b9b3bb06a8e0c998ead8e30362e7663c1ab2b85d3819bb855485663c92da4296270000000000000000000000001f9840a85d5af5bf1d1762f925bdaddc4201f984000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001b0622adacaeaa57d837c0c7b0814cdabc77dd8997985b753898ace8c7ac4f153a0e888a244c75b330dec0b609daf4a2d1734b7b8e8f95fac8f0845de2b4ee25a602000000000000000000000000000000000000000000000000000000000000727282747ecf0f1e02c1d91b592077453ccaa8aead781317f80b2e6de158b24f00000000000000000000000000000000000000000000039f95fe8e3fc3400000', + date: new Date('2021-04-15T00:54:44.000Z'), + from: '0xff59364722a4622a8d33623548926375b1b07767', + gasLimit: 380000, + gasPrice: '333000000000', + gasUsed: 261241, + hash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + index: 23, + nonce: '83376', + to: '0xe33c8e3a0d14a81f0dd7e174830089e82f65fc85', + value: '0', + }); + + await consumer.fn(job, mockOptions); + + expect(indexingBody).toEqual([ + { + index: { + _id: '60778fa4d2090c3c2ae307f7_5067df8b-f9cd-4a34-aee1-38d607100145', + }, + }, + { + appId: '5067df8b-f9cd-4a34-aee1-38d607100145', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 0, + relayedTradeValue: 0, + totalTradeCount: 1, + totalTradeValue: 12137.05, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xff59364722a4622a8d33623548926375b1b07767', + ], + }, + ]); + }); +}); diff --git a/src/fills/create-fills.js b/src/fills/create-fills.js index 6060900c7..edc84c234 100644 --- a/src/fills/create-fills.js +++ b/src/fills/create-fills.js @@ -9,6 +9,7 @@ const convertRelayerFees = require('./convert-relayer-fees'); const fetchUnknownAddressTypes = require('../addresses/fetch-unknown-address-types'); const hasProtocolFee = require('./has-protocol-fee'); const hasRelayerFees = require('./has-relayer-fees'); +const indexAppFills = require('../index/index-app-fills'); const indexFill = require('../index/index-fill'); const indexFillTraders = require('../index/index-fill-traders'); const indexTradedTokens = require('../index/index-traded-tokens'); @@ -65,6 +66,7 @@ const createFills = async (transaction, fills, { session } = {}) => { await indexFill(fillId, ms('30 seconds')); await indexTradedTokens(fill); await indexFillTraders(fill); + await indexAppFills(fill); if (hasProtocolFee(fill)) { await convertProtocolFee(fill, ms('30 seconds')); diff --git a/src/index/get-index-name.js b/src/index/get-index-name.js index 99f1813e5..b1f64e1a5 100644 --- a/src/index/get-index-name.js +++ b/src/index/get-index-name.js @@ -11,6 +11,10 @@ const getIndexName = index => { return process.env.INDEX_NAME_TRADER_FILLS || 'trader_fills'; } + if (index === 'app_fills') { + return process.env.INDEX_NAME_APP_FILLS || 'app_fills'; + } + if (index === 'network_metrics_daily') { return ( process.env.INDEX_NAME_NETWORK_METRICS_DAILY || 'network_metrics_daily' diff --git a/src/index/index-app-fills.js b/src/index/index-app-fills.js new file mode 100644 index 000000000..2f29c6ebb --- /dev/null +++ b/src/index/index-app-fills.js @@ -0,0 +1,53 @@ +const _ = require('lodash'); +const { JOB, QUEUE, FILL_ATTRIBUTION_TYPE } = require('../constants'); +const { publishJob } = require('../queues'); +const relayerRegistry = require('../relayers/relayer-registry'); + +const isOrderMatcher = relayerId => { + const relayer = _(relayerRegistry) + .values() + .find({ lookupId: relayerId }); + + return _.get(relayer, 'orderMatcher', false); +}; + +const calculateTradeCount = relayerId => { + if (isOrderMatcher(relayerId)) { + return 0.5; + } + + return 1; +}; + +const indexAppFills = async fill => { + if ( + !fill.attributions.some( + a => + a.type === FILL_ATTRIBUTION_TYPE.CONSUMER || + a.type === FILL_ATTRIBUTION_TYPE.RELAYER, + ) + ) { + return; + } + + const fillId = fill._id.toString(); + const value = _.get(fill, 'conversions.USD.amount'); + const tradeCount = calculateTradeCount(fill.relayerId); + const tradeValue = value ? value * tradeCount : null; + + publishJob(QUEUE.INDEXING, JOB.INDEX_APP_FILLS, { + attributions: fill.attributions.map(a => ({ + entityId: a.entityId, + type: a.type, + })), + fillDate: fill.date, + fillId, + maker: fill.maker, + taker: fill.taker, + tradeCount, + tradeValue, + transactionHash: fill.transactionHash, + }); +}; + +module.exports = indexAppFills; diff --git a/src/jobs/measure-fills/measure-fill.js b/src/jobs/measure-fills/measure-fill.js index 570fe9084..d2c92d78c 100644 --- a/src/jobs/measure-fills/measure-fill.js +++ b/src/jobs/measure-fills/measure-fill.js @@ -4,6 +4,7 @@ const { BASE_TOKENS, BASE_TOKEN_DECIMALS } = require('../../constants'); const formatTokenAmount = require('../../tokens/format-token-amount'); const getConversionRate = require('../../rates/get-conversion-rate'); const getMeasurableActor = require('./get-measurable-actor'); +const indexAppFills = require('../../index/index-app-fills'); const indexFillValue = require('../../index/index-fill-value'); const indexFillTraders = require('../../index/index-fill-traders'); const indexTradedTokens = require('../../index/index-traded-tokens'); @@ -80,6 +81,7 @@ const measureFill = async fill => { await indexFillValue(fill, totalValue); await indexFillTraders(fill); await indexTradedTokens(fill); + await indexAppFills(fill); }); }; diff --git a/src/jobs/measure-fills/measure-fill.test.js b/src/jobs/measure-fills/measure-fill.test.js index 162339e81..e07f22d74 100644 --- a/src/jobs/measure-fills/measure-fill.test.js +++ b/src/jobs/measure-fills/measure-fill.test.js @@ -78,4 +78,78 @@ describe('jobs/measure-fills/measure-fill', () => { '0x846d405f1ab48414c9a32f537c7b247e87e5770cc41dfe4255bc480710ec46f6', }); }); + + it('should update app_fills index after measurement when fill is associated with apps', async () => { + getConversionRate.mockResolvedValue(1); // 1 USD = 1USDC + + const Fill = getModel('Fill'); + const fill = new Fill({ + _id: '5e3e9abbb2227b1f0e73be5f', + status: 1, + attributions: [ + { + entityId: '5067df8b-f9cd-4a34-aee1-38d607100145', + type: 0, + }, + { entityId: 'dda3bbb3-333b-437a-ae96-150987c33617', type: 1 }, + ], + assets: [ + { + tokenResolved: true, + _id: '5e3e9abbb2227b1f0e73be61', + amount: 1674924411, + tokenAddress: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + bridgeAddress: '0x533344cfdf2a3e911e2cf4c6f5ed08e791f5355f', + bridgeData: + '0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + actor: 0, + }, + { + tokenResolved: true, + _id: '5e3e9abbb2227b1f0e73be60', + amount: 7.45216100214005e18, + tokenAddress: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + actor: 1, + }, + ], + blockHash: + '0x2d28446636839915b273eb93243dd6295f03d8117a55f0e6fb4f3cdc582d88e8', + blockNumber: 9441755, + date: '2020-02-08T11:21:14.000Z', + eventId: '5e3e9a370834b939aa28452c', + fees: [], + feeRecipient: '0x1000000000000000000000000000000000000011', + logIndex: 30, + maker: '0x533344cfdf2a3e911e2cf4c6f5ed08e791f5355f', + orderHash: + '0x0ccbf1bc14e19be3a4c422b18bb1c9870b5e0a75cfb75b694688b07380c13732', + protocolFee: 1500000000150000.0, + protocolVersion: 3, + relayerId: 31, + senderAddress: '0x4ef40d1bf0983899892946830abf99eca2dbc5ce', + taker: '0x4ef40d1bf0983899892946830abf99eca2dbc5ce', + transactionHash: + '0x846d405f1ab48414c9a32f537c7b247e87e5770cc41dfe4255bc480710ec46f6', + }); + + await measureFill(fill); + + expect(publishJob).toHaveBeenCalledWith('indexing', 'index-app-fills', { + attributions: [ + { + entityId: '5067df8b-f9cd-4a34-aee1-38d607100145', + type: 0, + }, + { entityId: 'dda3bbb3-333b-437a-ae96-150987c33617', type: 1 }, + ], + fillDate: new Date('2020-02-08T11:21:14.000Z'), + fillId: '5e3e9abbb2227b1f0e73be5f', + maker: '0x533344cfdf2a3e911e2cf4c6f5ed08e791f5355f', + taker: '0x4ef40d1bf0983899892946830abf99eca2dbc5ce', + tradeCount: 1, + tradeValue: 1674.924411, + transactionHash: + '0x846d405f1ab48414c9a32f537c7b247e87e5770cc41dfe4255bc480710ec46f6', + }); + }); });