From 45afdf78651e0e91063740c2ea91196ad7b6d45f Mon Sep 17 00:00:00 2001 From: Craig Bovis Date: Tue, 30 Mar 2021 22:03:30 -0500 Subject: [PATCH 1/5] Create job for populating app_fills index --- src/constants.js | 1 + src/consumers/index-app-fills/index.js | 119 +++++++++++++++++++++++++ src/fills/create-fills.js | 2 + src/index/index-app-fills.js | 40 +++++++++ 4 files changed, 162 insertions(+) create mode 100644 src/consumers/index-app-fills/index.js create mode 100644 src/index/index-app-fills.js diff --git a/src/constants.js b/src/constants.js index 55c6f6ae1..286939df8 100644 --- a/src/constants.js +++ b/src/constants.js @@ -86,6 +86,7 @@ module.exports = { FETCH_ADDRESS_TYPE: 'fetch-address-type', FETCH_TOKEN_METADATA: 'fetch-token-metadata', FETCH_TRANSACTION: 'fetch-transaction', + INDEX_APP_FILLS: 'index-app-fills', INDEX_FILL: 'index-fill', INDEX_FILL_PROTOCOL_FEE: 'index-fill-protocol-fee', INDEX_FILL_TRADERS: 'index-fill-traders', diff --git a/src/consumers/index-app-fills/index.js b/src/consumers/index-app-fills/index.js new file mode 100644 index 000000000..49b5fb5c9 --- /dev/null +++ b/src/consumers/index-app-fills/index.js @@ -0,0 +1,119 @@ +const _ = require('lodash'); + +const { JOB, QUEUE } = require('../../constants'); +const { publishJob } = require('../../queues'); +const elasticsearch = require('../../util/elasticsearch'); +const getAddressMetadata = require('../../addresses/get-address-metadata'); +const getIndexName = require('../../index/get-index-name'); +const getTransactionByHash = require('../../transactions/get-transaction-by-hash'); + +const indexAppFills = async (job, { logger }) => { + const delayJobProcessing = async () => { + await publishJob(QUEUE.INDEXING, JOB.INDEX_APP_FILLS, job.data, { + delay: 30000, + }); + }; + + const { + attributions, + fillDate, + fillId, + maker, + taker, + tradeCount, + tradeValue, + transactionHash, + } = job.data; + + const takerMetadata = await getAddressMetadata(taker); + + if (takerMetadata === null || takerMetadata.isContract === undefined) { + logger.warn(`taker address type is unknown: ${taker}`); + await delayJobProcessing(); + + return; + } + + let transaction; + + if (takerMetadata.isContract) { + transaction = await getTransactionByHash(transactionHash); + + if (transaction === null) { + logger.warn(`transaction has not been fetched: ${transactionHash}`); + await delayJobProcessing(); + + return; + } + } + + const traders = [maker, takerMetadata.isContract ? transaction.from : taker]; + + const appFills = _.reduce( + attributions, + (accumulator, current) => { + const existing = accumulator.find(x => x.entityId === current.entityId); + + if (existing) { + const replacement = { + ...existing, + relayedTradeCount: current.type === 0 ? tradeCount : 0, + relayedTradeValue: current.type === 0 ? tradeValue : 0, + totalTradeCount: existing.totalTradeCount + tradeCount, + totalTradeValue: existing.totalTradeValue + tradeValue, + }; + + return _.without(accumulator, existing).concat(replacement); + } + + return accumulator.concat({ + appId: current.entityId, + date: fillDate, + fillId, + relayedTradeCount: current.type === 0 ? tradeCount : 0, + relayedTradeValue: current.type === 0 ? tradeValue : 0, + totalTradeCount: tradeCount, + totalTradeValue: tradeValue, + traders, + updatedAt: new Date(Date.now()).toISOString(), + }); + }, + [], + ); + + if (appFills.length > 0) { + const requestBody = appFills + .map(appFill => + [ + JSON.stringify({ + index: { + _id: `${appFill.fillId}_${appFill.appId}`, + }, + }), + JSON.stringify(appFill), + ].join('\n'), + ) + .join('\n'); + + const result = await elasticsearch + .getClient() + .bulk({ body: `${requestBody}\n`, index: getIndexName('app_fills') }); + + if (result.body.errors === true) { + const errorMessage = _.get( + result, + 'body.items[0].index.error.reason', + `Indexing failed`, + ); + throw new Error(errorMessage); + } + } + + logger.info(`populated app_fills index for fill: ${fillId}`); +}; + +module.exports = { + fn: indexAppFills, + jobName: JOB.INDEX_APP_FILLS, + queueName: QUEUE.INDEXING, +}; diff --git a/src/fills/create-fills.js b/src/fills/create-fills.js index 6060900c7..edc84c234 100644 --- a/src/fills/create-fills.js +++ b/src/fills/create-fills.js @@ -9,6 +9,7 @@ const convertRelayerFees = require('./convert-relayer-fees'); const fetchUnknownAddressTypes = require('../addresses/fetch-unknown-address-types'); const hasProtocolFee = require('./has-protocol-fee'); const hasRelayerFees = require('./has-relayer-fees'); +const indexAppFills = require('../index/index-app-fills'); const indexFill = require('../index/index-fill'); const indexFillTraders = require('../index/index-fill-traders'); const indexTradedTokens = require('../index/index-traded-tokens'); @@ -65,6 +66,7 @@ const createFills = async (transaction, fills, { session } = {}) => { await indexFill(fillId, ms('30 seconds')); await indexTradedTokens(fill); await indexFillTraders(fill); + await indexAppFills(fill); if (hasProtocolFee(fill)) { await convertProtocolFee(fill, ms('30 seconds')); diff --git a/src/index/index-app-fills.js b/src/index/index-app-fills.js new file mode 100644 index 000000000..81e302cab --- /dev/null +++ b/src/index/index-app-fills.js @@ -0,0 +1,40 @@ +const _ = require('lodash'); +const { JOB, QUEUE } = require('../constants'); +const { publishJob } = require('../queues'); +const relayerRegistry = require('../relayers/relayer-registry'); + +const isOrderMatcher = relayerId => { + const relayer = _(relayerRegistry) + .values() + .find({ lookupId: relayerId }); + + return _.get(relayer, 'orderMatcher', false); +}; + +const calculateTradeCount = relayerId => { + if (isOrderMatcher(relayerId)) { + return 0.5; + } + + return 1; +}; + +const indexAppFills = async fill => { + const fillId = fill._id.toString(); + const value = _.get(fill, 'conversions.USD.amount'); + const tradeCount = calculateTradeCount(fill.relayerId); + const tradeValue = value ? value * tradeCount : null; + + publishJob(QUEUE.INDEXING, JOB.INDEX_APP_FILLS, { + attributions: fill.attributions, + fillDate: fill.date, + fillId, + maker: fill.maker, + taker: fill.taker, + tradeCount, + tradeValue, + transactionHash: fill.transactionHash, + }); +}; + +module.exports = indexAppFills; From 1e27edaeccc0c40467ab6eb5c493e1fddf0ccc9c Mon Sep 17 00:00:00 2001 From: Craig Bovis Date: Tue, 30 Mar 2021 22:21:41 -0500 Subject: [PATCH 2/5] Update app_fills index after measurement --- src/index/index-app-fills.js | 17 ++++- src/jobs/measure-fills/measure-fill.js | 2 + src/jobs/measure-fills/measure-fill.test.js | 74 +++++++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/src/index/index-app-fills.js b/src/index/index-app-fills.js index 81e302cab..2f29c6ebb 100644 --- a/src/index/index-app-fills.js +++ b/src/index/index-app-fills.js @@ -1,5 +1,5 @@ const _ = require('lodash'); -const { JOB, QUEUE } = require('../constants'); +const { JOB, QUEUE, FILL_ATTRIBUTION_TYPE } = require('../constants'); const { publishJob } = require('../queues'); const relayerRegistry = require('../relayers/relayer-registry'); @@ -20,13 +20,26 @@ const calculateTradeCount = relayerId => { }; const indexAppFills = async fill => { + if ( + !fill.attributions.some( + a => + a.type === FILL_ATTRIBUTION_TYPE.CONSUMER || + a.type === FILL_ATTRIBUTION_TYPE.RELAYER, + ) + ) { + return; + } + const fillId = fill._id.toString(); const value = _.get(fill, 'conversions.USD.amount'); const tradeCount = calculateTradeCount(fill.relayerId); const tradeValue = value ? value * tradeCount : null; publishJob(QUEUE.INDEXING, JOB.INDEX_APP_FILLS, { - attributions: fill.attributions, + attributions: fill.attributions.map(a => ({ + entityId: a.entityId, + type: a.type, + })), fillDate: fill.date, fillId, maker: fill.maker, diff --git a/src/jobs/measure-fills/measure-fill.js b/src/jobs/measure-fills/measure-fill.js index 570fe9084..d2c92d78c 100644 --- a/src/jobs/measure-fills/measure-fill.js +++ b/src/jobs/measure-fills/measure-fill.js @@ -4,6 +4,7 @@ const { BASE_TOKENS, BASE_TOKEN_DECIMALS } = require('../../constants'); const formatTokenAmount = require('../../tokens/format-token-amount'); const getConversionRate = require('../../rates/get-conversion-rate'); const getMeasurableActor = require('./get-measurable-actor'); +const indexAppFills = require('../../index/index-app-fills'); const indexFillValue = require('../../index/index-fill-value'); const indexFillTraders = require('../../index/index-fill-traders'); const indexTradedTokens = require('../../index/index-traded-tokens'); @@ -80,6 +81,7 @@ const measureFill = async fill => { await indexFillValue(fill, totalValue); await indexFillTraders(fill); await indexTradedTokens(fill); + await indexAppFills(fill); }); }; diff --git a/src/jobs/measure-fills/measure-fill.test.js b/src/jobs/measure-fills/measure-fill.test.js index 162339e81..e07f22d74 100644 --- a/src/jobs/measure-fills/measure-fill.test.js +++ b/src/jobs/measure-fills/measure-fill.test.js @@ -78,4 +78,78 @@ describe('jobs/measure-fills/measure-fill', () => { '0x846d405f1ab48414c9a32f537c7b247e87e5770cc41dfe4255bc480710ec46f6', }); }); + + it('should update app_fills index after measurement when fill is associated with apps', async () => { + getConversionRate.mockResolvedValue(1); // 1 USD = 1USDC + + const Fill = getModel('Fill'); + const fill = new Fill({ + _id: '5e3e9abbb2227b1f0e73be5f', + status: 1, + attributions: [ + { + entityId: '5067df8b-f9cd-4a34-aee1-38d607100145', + type: 0, + }, + { entityId: 'dda3bbb3-333b-437a-ae96-150987c33617', type: 1 }, + ], + assets: [ + { + tokenResolved: true, + _id: '5e3e9abbb2227b1f0e73be61', + amount: 1674924411, + tokenAddress: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + bridgeAddress: '0x533344cfdf2a3e911e2cf4c6f5ed08e791f5355f', + bridgeData: + '0x000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + actor: 0, + }, + { + tokenResolved: true, + _id: '5e3e9abbb2227b1f0e73be60', + amount: 7.45216100214005e18, + tokenAddress: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + actor: 1, + }, + ], + blockHash: + '0x2d28446636839915b273eb93243dd6295f03d8117a55f0e6fb4f3cdc582d88e8', + blockNumber: 9441755, + date: '2020-02-08T11:21:14.000Z', + eventId: '5e3e9a370834b939aa28452c', + fees: [], + feeRecipient: '0x1000000000000000000000000000000000000011', + logIndex: 30, + maker: '0x533344cfdf2a3e911e2cf4c6f5ed08e791f5355f', + orderHash: + '0x0ccbf1bc14e19be3a4c422b18bb1c9870b5e0a75cfb75b694688b07380c13732', + protocolFee: 1500000000150000.0, + protocolVersion: 3, + relayerId: 31, + senderAddress: '0x4ef40d1bf0983899892946830abf99eca2dbc5ce', + taker: '0x4ef40d1bf0983899892946830abf99eca2dbc5ce', + transactionHash: + '0x846d405f1ab48414c9a32f537c7b247e87e5770cc41dfe4255bc480710ec46f6', + }); + + await measureFill(fill); + + expect(publishJob).toHaveBeenCalledWith('indexing', 'index-app-fills', { + attributions: [ + { + entityId: '5067df8b-f9cd-4a34-aee1-38d607100145', + type: 0, + }, + { entityId: 'dda3bbb3-333b-437a-ae96-150987c33617', type: 1 }, + ], + fillDate: new Date('2020-02-08T11:21:14.000Z'), + fillId: '5e3e9abbb2227b1f0e73be5f', + maker: '0x533344cfdf2a3e911e2cf4c6f5ed08e791f5355f', + taker: '0x4ef40d1bf0983899892946830abf99eca2dbc5ce', + tradeCount: 1, + tradeValue: 1674.924411, + transactionHash: + '0x846d405f1ab48414c9a32f537c7b247e87e5770cc41dfe4255bc480710ec46f6', + }); + }); }); From 5f48d5c17c0e6f90f0f3fcaa055f07716cc53c61 Mon Sep 17 00:00:00 2001 From: Craig Bovis Date: Wed, 14 Apr 2021 19:37:06 -0500 Subject: [PATCH 3/5] Refactor --- src/consumers/index-app-fills/index.js | 29 +++++++++++++++----------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/consumers/index-app-fills/index.js b/src/consumers/index-app-fills/index.js index 49b5fb5c9..8fe2754f6 100644 --- a/src/consumers/index-app-fills/index.js +++ b/src/consumers/index-app-fills/index.js @@ -1,6 +1,6 @@ const _ = require('lodash'); -const { JOB, QUEUE } = require('../../constants'); +const { JOB, QUEUE, FILL_ATTRIBUTION_TYPE } = require('../../constants'); const { publishJob } = require('../../queues'); const elasticsearch = require('../../util/elasticsearch'); const getAddressMetadata = require('../../addresses/get-address-metadata'); @@ -49,16 +49,22 @@ const indexAppFills = async (job, { logger }) => { const traders = [maker, takerMetadata.isContract ? transaction.from : taker]; - const appFills = _.reduce( - attributions, - (accumulator, current) => { + const appFills = attributions + .filter( + attribution => + attribution.type === FILL_ATTRIBUTION_TYPE.CONSUMER || + attribution.type === FILL_ATTRIBUTION_TYPE.RELAYER, + ) + .reduce((accumulator, current) => { const existing = accumulator.find(x => x.entityId === current.entityId); if (existing) { const replacement = { ...existing, - relayedTradeCount: current.type === 0 ? tradeCount : 0, - relayedTradeValue: current.type === 0 ? tradeValue : 0, + relayedTradeCount: + current.type === FILL_ATTRIBUTION_TYPE.RELAYER ? tradeCount : 0, + relayedTradeValue: + current.type === FILL_ATTRIBUTION_TYPE.RELAYER ? tradeValue : 0, totalTradeCount: existing.totalTradeCount + tradeCount, totalTradeValue: existing.totalTradeValue + tradeValue, }; @@ -70,16 +76,15 @@ const indexAppFills = async (job, { logger }) => { appId: current.entityId, date: fillDate, fillId, - relayedTradeCount: current.type === 0 ? tradeCount : 0, - relayedTradeValue: current.type === 0 ? tradeValue : 0, + relayedTradeCount: + current.type === FILL_ATTRIBUTION_TYPE.RELAYER ? tradeCount : 0, + relayedTradeValue: + current.type === FILL_ATTRIBUTION_TYPE.RELAYER ? tradeValue : 0, totalTradeCount: tradeCount, totalTradeValue: tradeValue, traders, - updatedAt: new Date(Date.now()).toISOString(), }); - }, - [], - ); + }, []); if (appFills.length > 0) { const requestBody = appFills From d397a9c01329e0397aa26affd833a6701caf69b9 Mon Sep 17 00:00:00 2001 From: Craig Bovis Date: Wed, 14 Apr 2021 20:05:33 -0500 Subject: [PATCH 4/5] Add index name to lookup --- src/index/get-index-name.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/index/get-index-name.js b/src/index/get-index-name.js index 99f1813e5..b1f64e1a5 100644 --- a/src/index/get-index-name.js +++ b/src/index/get-index-name.js @@ -11,6 +11,10 @@ const getIndexName = index => { return process.env.INDEX_NAME_TRADER_FILLS || 'trader_fills'; } + if (index === 'app_fills') { + return process.env.INDEX_NAME_APP_FILLS || 'app_fills'; + } + if (index === 'network_metrics_daily') { return ( process.env.INDEX_NAME_NETWORK_METRICS_DAILY || 'network_metrics_daily' From 677674f8b43ec5447fca64f218ca14255fa985c3 Mon Sep 17 00:00:00 2001 From: Craig Bovis Date: Wed, 14 Apr 2021 21:33:41 -0500 Subject: [PATCH 5/5] Fix some issues and add tests --- src/consumers/index-app-fills/index.js | 99 ++- src/consumers/index-app-fills/index.test.js | 659 ++++++++++++++++++++ 2 files changed, 705 insertions(+), 53 deletions(-) create mode 100644 src/consumers/index-app-fills/index.test.js diff --git a/src/consumers/index-app-fills/index.js b/src/consumers/index-app-fills/index.js index 8fe2754f6..8add9a95e 100644 --- a/src/consumers/index-app-fills/index.js +++ b/src/consumers/index-app-fills/index.js @@ -49,69 +49,62 @@ const indexAppFills = async (job, { logger }) => { const traders = [maker, takerMetadata.isContract ? transaction.from : taker]; - const appFills = attributions + const appFills = _(attributions) .filter( attribution => attribution.type === FILL_ATTRIBUTION_TYPE.CONSUMER || attribution.type === FILL_ATTRIBUTION_TYPE.RELAYER, ) - .reduce((accumulator, current) => { - const existing = accumulator.find(x => x.entityId === current.entityId); - - if (existing) { - const replacement = { - ...existing, - relayedTradeCount: - current.type === FILL_ATTRIBUTION_TYPE.RELAYER ? tradeCount : 0, - relayedTradeValue: - current.type === FILL_ATTRIBUTION_TYPE.RELAYER ? tradeValue : 0, - totalTradeCount: existing.totalTradeCount + tradeCount, - totalTradeValue: existing.totalTradeValue + tradeValue, - }; - - return _.without(accumulator, existing).concat(replacement); - } - - return accumulator.concat({ - appId: current.entityId, + .map(attribution => attribution.entityId) + .uniq() + .map(appId => { + const isRelayer = attributions.some( + a => a.entityId === appId && a.type === FILL_ATTRIBUTION_TYPE.RELAYER, + ); + + return { + appId, date: fillDate, fillId, - relayedTradeCount: - current.type === FILL_ATTRIBUTION_TYPE.RELAYER ? tradeCount : 0, - relayedTradeValue: - current.type === FILL_ATTRIBUTION_TYPE.RELAYER ? tradeValue : 0, + relayedTradeCount: isRelayer ? tradeCount : 0, + relayedTradeValue: isRelayer || !tradeValue ? tradeValue : 0, totalTradeCount: tradeCount, totalTradeValue: tradeValue, traders, - }); - }, []); - - if (appFills.length > 0) { - const requestBody = appFills - .map(appFill => - [ - JSON.stringify({ - index: { - _id: `${appFill.fillId}_${appFill.appId}`, - }, - }), - JSON.stringify(appFill), - ].join('\n'), - ) - .join('\n'); - - const result = await elasticsearch - .getClient() - .bulk({ body: `${requestBody}\n`, index: getIndexName('app_fills') }); - - if (result.body.errors === true) { - const errorMessage = _.get( - result, - 'body.items[0].index.error.reason', - `Indexing failed`, - ); - throw new Error(errorMessage); - } + }; + }) + .value(); + + if (appFills.length === 0) { + logger.info(`skipped app_fills indexing for fill: ${fillId}`); + + return; + } + + const requestBody = appFills + .map(appFill => + [ + JSON.stringify({ + index: { + _id: `${appFill.fillId}_${appFill.appId}`, + }, + }), + JSON.stringify(appFill), + ].join('\n'), + ) + .join('\n'); + + const result = await elasticsearch + .getClient() + .bulk({ body: `${requestBody}\n`, index: getIndexName('app_fills') }); + + if (result.body.errors === true) { + const errorMessage = _.get( + result, + 'body.items[0].index.error.reason', + `Indexing failed`, + ); + throw new Error(errorMessage); } logger.info(`populated app_fills index for fill: ${fillId}`); diff --git a/src/consumers/index-app-fills/index.test.js b/src/consumers/index-app-fills/index.test.js new file mode 100644 index 000000000..870d467b8 --- /dev/null +++ b/src/consumers/index-app-fills/index.test.js @@ -0,0 +1,659 @@ +const { Client } = require('@elastic/elasticsearch'); +const Mock = require('@elastic/elasticsearch-mock'); + +const { getModel } = require('../../model'); +const { + mockLogger, + resetDb, + setupDb, + tearDownDb, +} = require('../../test-utils'); +const { publishJob } = require('../../queues'); +const elasticsearch = require('../../util/elasticsearch'); +const consumer = require('.'); + +jest.mock('../../queues'); +jest.mock('../../util/elasticsearch'); + +const elasticsearchMock = new Mock(); +const mockOptions = { + logger: mockLogger, +}; + +beforeAll(async () => { + const client = new Client({ + node: 'http://localhost:9200', + Connection: elasticsearchMock.getConnection(), + }); + + elasticsearch.getClient.mockReturnValue(client); + await setupDb(); +}, 30000); + +afterEach(async () => { + jest.clearAllMocks(); + await resetDb(); + elasticsearchMock.clearAll(); +}, 30000); + +afterAll(async () => { + await tearDownDb(); +}, 30000); + +describe('consumers/index-app-fills', () => { + it('should consume indexing queue', () => { + expect(consumer.queueName).toBe('indexing'); + }); + + it('should consume index-app-fills jobs', () => { + expect(consumer.jobName).toBe('index-app-fills'); + }); + + it('should delay job processing when taker type unknown', async () => { + const job = { + data: { + attributions: [], + fillDate: new Date('2020-08-02T07:47:28Z'), + fillId: '5f267c7b545e125452c56e14', + maker: '0x903153f55770b7668a497180f7fa93471545ffe2', + taker: '0xd0f8715fda0c1b564c2087315fb55804eaf1fae9', + tradeCount: 1, + tradeValue: 500, + transactionHash: + '0x165a1b8f4fbcf089e48435c2efef12dab5223c560f067f932c8efc3d7c6d74eb', + }, + }; + + await consumer.fn(job, mockOptions); + + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(mockLogger.warn).toHaveBeenCalledWith( + 'taker address type is unknown: 0xd0f8715fda0c1b564c2087315fb55804eaf1fae9', + ); + + expect(publishJob).toHaveBeenCalledTimes(1); + expect(publishJob).toHaveBeenCalledWith( + 'indexing', + 'index-app-fills', + job.data, + { delay: 30000 }, + ); + }); + + it('should delay job processing when taker is contract and associated transaction not fetched', async () => { + const job = { + data: { + attributions: [], + fillDate: new Date('2020-08-02T07:47:28Z'), + fillId: '5f267c7b545e125452c56e14', + maker: '0x903153f55770b7668a497180f7fa93471545ffe2', + taker: '0xd0f8715fda0c1b564c2087315fb55804eaf1fae9', + tradeCount: 1, + tradeValue: 500, + transactionHash: + '0x165a1b8f4fbcf089e48435c2efef12dab5223c560f067f932c8efc3d7c6d74eb', + }, + }; + + const AddressMetadata = getModel('AddressMetadata'); + + await AddressMetadata.create({ + address: '0xd0f8715fda0c1b564c2087315fb55804eaf1fae9', + isContract: true, + }); + + await consumer.fn(job, mockOptions); + + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(mockLogger.warn).toHaveBeenCalledWith( + 'transaction has not been fetched: 0x165a1b8f4fbcf089e48435c2efef12dab5223c560f067f932c8efc3d7c6d74eb', + ); + + expect(publishJob).toHaveBeenCalledTimes(1); + expect(publishJob).toHaveBeenCalledWith( + 'indexing', + 'index-app-fills', + job.data, + { delay: 30000 }, + ); + }); + + it('should index a single document when relayer and consumer are the same', async () => { + const job = { + data: { + attributions: [ + { + entityId: '052b4862-2142-4532-bdc0-416814b0a5fe', + type: 0, + }, + { + entityId: '052b4862-2142-4532-bdc0-416814b0a5fe', + type: 1, + }, + { + entityId: 'bdb40272-89f6-4972-b88c-b9baf0ef5410', + type: 2, + }, + ], + fillDate: new Date('2021-04-15T00:54:44.000Z'), + fillId: '60778fa4d2090c3c2ae307f7', + maker: '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + taker: '0xd965952823153e5cbc611be87e8322cfc329f056', + tradeCount: 1, + tradeValue: 12137.05, + transactionHash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + }, + }; + + let indexingBody; + + elasticsearchMock.add( + { + method: 'POST', + path: '/app_fills/_bulk', + }, + ({ body }) => { + indexingBody = body; + + return { status: 'ok' }; + }, + ); + + const AddressMetadata = getModel('AddressMetadata'); + + await AddressMetadata.create({ + address: '0xd965952823153e5cbc611be87e8322cfc329f056', + isContract: false, + }); + + await consumer.fn(job, mockOptions); + + expect(indexingBody).toEqual([ + { + index: { + _id: '60778fa4d2090c3c2ae307f7_052b4862-2142-4532-bdc0-416814b0a5fe', + }, + }, + { + appId: '052b4862-2142-4532-bdc0-416814b0a5fe', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 1, + relayedTradeValue: 12137.05, + totalTradeCount: 1, + totalTradeValue: 12137.05, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xd965952823153e5cbc611be87e8322cfc329f056', + ], + }, + ]); + }); + + it('should index two documents when relayer and consumer are different', async () => { + const job = { + data: { + attributions: [ + { + entityId: '052b4862-2142-4532-bdc0-416814b0a5fe', + type: 0, + }, + { + entityId: '5067df8b-f9cd-4a34-aee1-38d607100145', + type: 1, + }, + { + entityId: 'bdb40272-89f6-4972-b88c-b9baf0ef5410', + type: 2, + }, + ], + fillDate: new Date('2021-04-15T00:54:44.000Z'), + fillId: '60778fa4d2090c3c2ae307f7', + maker: '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + taker: '0xd965952823153e5cbc611be87e8322cfc329f056', + tradeCount: 1, + tradeValue: 12137.05, + transactionHash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + }, + }; + + let indexingBody; + + elasticsearchMock.add( + { + method: 'POST', + path: '/app_fills/_bulk', + }, + ({ body }) => { + indexingBody = body; + + return { status: 'ok' }; + }, + ); + + const AddressMetadata = getModel('AddressMetadata'); + + await AddressMetadata.create({ + address: '0xd965952823153e5cbc611be87e8322cfc329f056', + isContract: false, + }); + + await consumer.fn(job, mockOptions); + + expect(indexingBody).toEqual([ + { + index: { + _id: '60778fa4d2090c3c2ae307f7_052b4862-2142-4532-bdc0-416814b0a5fe', + }, + }, + { + appId: '052b4862-2142-4532-bdc0-416814b0a5fe', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 1, + relayedTradeValue: 12137.05, + totalTradeCount: 1, + totalTradeValue: 12137.05, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xd965952823153e5cbc611be87e8322cfc329f056', + ], + }, + { + index: { + _id: '60778fa4d2090c3c2ae307f7_5067df8b-f9cd-4a34-aee1-38d607100145', + }, + }, + { + appId: '5067df8b-f9cd-4a34-aee1-38d607100145', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 0, + relayedTradeValue: 0, + totalTradeCount: 1, + totalTradeValue: 12137.05, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xd965952823153e5cbc611be87e8322cfc329f056', + ], + }, + ]); + }); + + it('should skip indexing when fill does not have any app attributions', async () => { + const job = { + data: { + attributions: [ + { + entityId: 'bdb40272-89f6-4972-b88c-b9baf0ef5410', + type: 2, + }, + ], + fillDate: new Date('2021-04-15T00:54:44.000Z'), + fillId: '60778fa4d2090c3c2ae307f7', + maker: '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + taker: '0xd965952823153e5cbc611be87e8322cfc329f056', + tradeCount: 1, + tradeValue: 12137.05, + transactionHash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + }, + }; + + let indexingBody; + + elasticsearchMock.add( + { + method: 'POST', + path: '/app_fills/_bulk', + }, + ({ body }) => { + indexingBody = body; + + return { status: 'ok' }; + }, + ); + + const AddressMetadata = getModel('AddressMetadata'); + + await AddressMetadata.create({ + address: '0xd965952823153e5cbc611be87e8322cfc329f056', + isContract: false, + }); + + await consumer.fn(job, mockOptions); + + expect(indexingBody).toBeUndefined(); + expect(mockLogger.info).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + 'skipped app_fills indexing for fill: 60778fa4d2090c3c2ae307f7', + ); + }); + + it('should index single document when fill only has a relayer', async () => { + const job = { + data: { + attributions: [ + { + entityId: '052b4862-2142-4532-bdc0-416814b0a5fe', + type: 0, + }, + { + entityId: 'bdb40272-89f6-4972-b88c-b9baf0ef5410', + type: 2, + }, + ], + fillDate: new Date('2021-04-15T00:54:44.000Z'), + fillId: '60778fa4d2090c3c2ae307f7', + maker: '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + taker: '0xd965952823153e5cbc611be87e8322cfc329f056', + tradeCount: 1, + tradeValue: 12137.05, + transactionHash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + }, + }; + + let indexingBody; + + elasticsearchMock.add( + { + method: 'POST', + path: '/app_fills/_bulk', + }, + ({ body }) => { + indexingBody = body; + + return { status: 'ok' }; + }, + ); + + const AddressMetadata = getModel('AddressMetadata'); + + await AddressMetadata.create({ + address: '0xd965952823153e5cbc611be87e8322cfc329f056', + isContract: false, + }); + + await consumer.fn(job, mockOptions); + + expect(indexingBody).toEqual([ + { + index: { + _id: '60778fa4d2090c3c2ae307f7_052b4862-2142-4532-bdc0-416814b0a5fe', + }, + }, + { + appId: '052b4862-2142-4532-bdc0-416814b0a5fe', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 1, + relayedTradeValue: 12137.05, + totalTradeCount: 1, + totalTradeValue: 12137.05, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xd965952823153e5cbc611be87e8322cfc329f056', + ], + }, + ]); + + expect(mockLogger.info).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + 'populated app_fills index for fill: 60778fa4d2090c3c2ae307f7', + ); + }); + + it('should index single document when fill only has a consumer', async () => { + const job = { + data: { + attributions: [ + { + entityId: '5067df8b-f9cd-4a34-aee1-38d607100145', + type: 1, + }, + { + entityId: 'bdb40272-89f6-4972-b88c-b9baf0ef5410', + type: 2, + }, + ], + fillDate: new Date('2021-04-15T00:54:44.000Z'), + fillId: '60778fa4d2090c3c2ae307f7', + maker: '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + taker: '0xd965952823153e5cbc611be87e8322cfc329f056', + tradeCount: 1, + tradeValue: 12137.05, + transactionHash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + }, + }; + + let indexingBody; + + elasticsearchMock.add( + { + method: 'POST', + path: '/app_fills/_bulk', + }, + ({ body }) => { + indexingBody = body; + + return { status: 'ok' }; + }, + ); + + const AddressMetadata = getModel('AddressMetadata'); + + await AddressMetadata.create({ + address: '0xd965952823153e5cbc611be87e8322cfc329f056', + isContract: false, + }); + + await consumer.fn(job, mockOptions); + + expect(indexingBody).toEqual([ + { + index: { + _id: '60778fa4d2090c3c2ae307f7_5067df8b-f9cd-4a34-aee1-38d607100145', + }, + }, + { + appId: '5067df8b-f9cd-4a34-aee1-38d607100145', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 0, + relayedTradeValue: 0, + totalTradeCount: 1, + totalTradeValue: 12137.05, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xd965952823153e5cbc611be87e8322cfc329f056', + ], + }, + ]); + + expect(mockLogger.info).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + 'populated app_fills index for fill: 60778fa4d2090c3c2ae307f7', + ); + }); + + it('should index without trade value if unavailable', async () => { + const job = { + data: { + attributions: [ + { + entityId: '052b4862-2142-4532-bdc0-416814b0a5fe', + type: 0, + }, + { + entityId: '5067df8b-f9cd-4a34-aee1-38d607100145', + type: 1, + }, + { + entityId: 'bdb40272-89f6-4972-b88c-b9baf0ef5410', + type: 2, + }, + ], + fillDate: new Date('2021-04-15T00:54:44.000Z'), + fillId: '60778fa4d2090c3c2ae307f7', + maker: '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + taker: '0xd965952823153e5cbc611be87e8322cfc329f056', + tradeCount: 1, + tradeValue: undefined, + transactionHash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + }, + }; + + let indexingBody; + + elasticsearchMock.add( + { + method: 'POST', + path: '/app_fills/_bulk', + }, + ({ body }) => { + indexingBody = body; + + return { status: 'ok' }; + }, + ); + + const AddressMetadata = getModel('AddressMetadata'); + + await AddressMetadata.create({ + address: '0xd965952823153e5cbc611be87e8322cfc329f056', + isContract: false, + }); + + await consumer.fn(job, mockOptions); + + expect(indexingBody).toEqual([ + { + index: { + _id: '60778fa4d2090c3c2ae307f7_052b4862-2142-4532-bdc0-416814b0a5fe', + }, + }, + { + appId: '052b4862-2142-4532-bdc0-416814b0a5fe', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 1, + totalTradeCount: 1, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xd965952823153e5cbc611be87e8322cfc329f056', + ], + }, + { + index: { + _id: '60778fa4d2090c3c2ae307f7_5067df8b-f9cd-4a34-aee1-38d607100145', + }, + }, + { + appId: '5067df8b-f9cd-4a34-aee1-38d607100145', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 0, + totalTradeCount: 1, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xd965952823153e5cbc611be87e8322cfc329f056', + ], + }, + ]); + + expect(mockLogger.info).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + 'populated app_fills index for fill: 60778fa4d2090c3c2ae307f7', + ); + }); + + it('should index transaction sender as taker when taker address is a contract', async () => { + const job = { + data: { + attributions: [ + { + entityId: '5067df8b-f9cd-4a34-aee1-38d607100145', + type: 1, + }, + ], + fillDate: new Date('2021-04-15T00:54:44.000Z'), + fillId: '60778fa4d2090c3c2ae307f7', + maker: '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + taker: '0xd965952823153e5cbc611be87e8322cfc329f056', + tradeCount: 1, + tradeValue: 12137.05, + transactionHash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + }, + }; + + const AddressMetadata = getModel('AddressMetadata'); + const Transaction = getModel('Transaction'); + + let indexingBody; + + elasticsearchMock.add( + { + method: 'POST', + path: '/app_fills/_bulk', + }, + ({ body }) => { + indexingBody = body; + + return { status: 'ok' }; + }, + ); + + await AddressMetadata.create({ + address: '0xd965952823153e5cbc611be87e8322cfc329f056', + isContract: true, + }); + + await Transaction.create({ + blockHash: + '0xb7a0059c1aff2e84f7d5b75dd29b57325bd6a8dd0469240689cc52df1df1146a', + blockNumber: 10893374, + data: + '0x000d070a11ced27fd19c7a2bc70fd72adb55bb7025014e00000000000000000000d3d2e2692501a5c9ca623199d38826e513033a17000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000001f9840a85d5af5bf1d1762f925bdaddc4201f984000000000000000000000000e33c8e3a0d14a81f0dd7e174830089e82f65fc85000000000000000000000000000000000000000000000003255b68cdf304cdc40000000000000000000000000000000000000000000000031e171c6104e502420000000000000000000000000000000000000000000000101ad5ea23c2190000014e000000000000000000005e8d405cbc564473d85a9a31fbfca76167d6997800000000000000000000000086003b044f70dac0abc80ac8957305b6370893ed00000000000000000000000000000000000000000000001043561a882930000000000000000000000000000000000000000000000000039f985e76e5df87c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005f6a05b9b3bb06a8e0c998ead8e30362e7663c1ab2b85d3819bb855485663c92da4296270000000000000000000000001f9840a85d5af5bf1d1762f925bdaddc4201f984000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001b0622adacaeaa57d837c0c7b0814cdabc77dd8997985b753898ace8c7ac4f153a0e888a244c75b330dec0b609daf4a2d1734b7b8e8f95fac8f0845de2b4ee25a602000000000000000000000000000000000000000000000000000000000000727282747ecf0f1e02c1d91b592077453ccaa8aead781317f80b2e6de158b24f00000000000000000000000000000000000000000000039f95fe8e3fc3400000', + date: new Date('2021-04-15T00:54:44.000Z'), + from: '0xff59364722a4622a8d33623548926375b1b07767', + gasLimit: 380000, + gasPrice: '333000000000', + gasUsed: 261241, + hash: + '0x68451bd19ef11a32fe8f801b90d775094f7a8cc1d1ef365892addd1c5dccf13e', + index: 23, + nonce: '83376', + to: '0xe33c8e3a0d14a81f0dd7e174830089e82f65fc85', + value: '0', + }); + + await consumer.fn(job, mockOptions); + + expect(indexingBody).toEqual([ + { + index: { + _id: '60778fa4d2090c3c2ae307f7_5067df8b-f9cd-4a34-aee1-38d607100145', + }, + }, + { + appId: '5067df8b-f9cd-4a34-aee1-38d607100145', + fillId: '60778fa4d2090c3c2ae307f7', + date: '2021-04-15T00:54:44.000Z', + relayedTradeCount: 0, + relayedTradeValue: 0, + totalTradeCount: 1, + totalTradeValue: 12137.05, + traders: [ + '0x73e02eaab68a41ea63bdae9dbd4b7678827b2352', + '0xff59364722a4622a8d33623548926375b1b07767', + ], + }, + ]); + }); +});