Filter out duplicate bridge transfer events (#453)

Author: cbovis

src/consumers/create-transformed-erc20-event-fills/index.js

@@ -13,6 +13,10 @@ const Fill = require('../../model/fill');
 const getTransactionByHash = require('../../transactions/get-transaction-by-hash');
 const withTransaction = require('../../util/with-transaction');
 
+const dedupeBridgeEvents = events => {
+  return _.uniqWith(events, (a, b) => _.isEqual(a.data, b.data));
+};
+
 const createTransformedERC20EventFills = async (job, { logger }) => {
   const { eventId } = job.data;
 
@@ -134,7 +138,7 @@ const createTransformedERC20EventFills = async (job, { logger }) => {
    * event is related to the TransformedERC20 event being processed and will use the TransformedERC20
    * event to dictate the token and taker addresses.
    */
-  const fills = bridgeEvents.map(bridgeEvent => ({
+  const fills = dedupeBridgeEvents(bridgeEvents).map(bridgeEvent => ({
     _id: bridgeEvent._id,
     affiliateAddress: transaction.affiliateAddress.toLowerCase(),
     assets: [

src/consumers/create-transformed-erc20-event-fills/index.test.js

@@ -818,4 +818,96 @@ describe('consumers/create-transformed-erc20-event-fills', () => {
 
     expect(fills).toHaveLength(1);
   });
+
+  /*
+    This scenario handles an edge-case whereby some trades will emit two ERC20BridgeTransfer
+    events for a single transfer. If the events are not deduped then volume gets double counted.
+  */
+  it('should dedupe ERC20BridgeTransfer events before creating fills', async () => {
+    await Event.create([
+      {
+        _id: '5f4e754b6609e17174cbf1d2',
+        protocolVersion: 3,
+        blockNumber: 10776336,
+        data: {
+          inputToken: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48',
+          inputTokenAmount: '7108136500',
+          outputToken: '0xdac17f958d2ee523a2206206994597c13d831ec7',
+          outputTokenAmount: '7113021184',
+          taker: '0x9b43818e253baed56e14a96d677347a06c77c6c1',
+        },
+        dateIngested: new Date('2020-09-01T16:22:35.629Z'),
+        logIndex: 181,
+        transactionHash:
+          '0x70462e739348dfab76384297b7461293800163691b28b3ba5413c07465b6b6f1',
+        type: 'TransformedERC20',
+      },
+      {
+        _id: '5f4e7552a21d432dbead93f7',
+        blockNumber: 10776336,
+        data: {
+          from: '0xAF76A7c57B441501bb7A6354728e3F5e13872139',
+          fromToken: '0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48',
+          fromTokenAmount: '7108136500',
+          to: '0x22F9dCF4647084d6C31b2765F6910cd85C178C18',
+          toToken: '0xdAC17F958D2ee523a2206206994597C13D831ec7',
+          toTokenAmount: '7113021184',
+        },
+        logIndex: 177,
+        transactionHash:
+          '0x70462e739348dfab76384297b7461293800163691b28b3ba5413c07465b6b6f1',
+        type: 'ERC20BridgeTransfer',
+        protocolVersion: 3,
+        dateIngested: new Date('2020-09-01T16:22:42.946Z'),
+      },
+      {
+        _id: '5f4e7552a21d432dbead93f8',
+        blockNumber: 10776336,
+        data: {
+          from: '0xAF76A7c57B441501bb7A6354728e3F5e13872139',
+          fromToken: '0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48',
+          fromTokenAmount: '7108136500',
+          to: '0x22F9dCF4647084d6C31b2765F6910cd85C178C18',
+          toToken: '0xdAC17F958D2ee523a2206206994597C13D831ec7',
+          toTokenAmount: '7113021184',
+        },
+        logIndex: 179,
+        transactionHash:
+          '0x70462e739348dfab76384297b7461293800163691b28b3ba5413c07465b6b6f1',
+        type: 'ERC20BridgeTransfer',
+        protocolVersion: 3,
+        dateIngested: new Date('2020-09-01T16:22:42.946Z'),
+      },
+    ]);
+
+    await Transaction.create({
+      _id: '5f4e7552a21d432dbead93f9',
+      affiliateAddress: '0x86003b044f70dac0abc80ac8957305b6370893ed',
+      blockHash:
+        '0xa9bd4f1f3fa5d4f48aec53ed1b48220f24b6852a47c5a3fa914f955170d9171e',
+      blockNumber: 10776336,
+      data:
+        '0x415565b0000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb48000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec700000000000000000000000000000000000000000000000000000001a7ad8e3400000000000000000000000000000000000000000000000000000001a3baba9b00000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000005800000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000004e000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb48000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec700000000000000000000000000000000000000000000000000000000000000e0000000000000000000000000000000000000000000000000000000000000042000000000000000000000000000000000000000000000000000000000000004a000000000000000000000000000000000000000000000000000000001a7ad8e3400000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000020000000000000000000000000af76a7c57b441501bb7a6354728e3f5e1387213900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001a3baba9c00000000000000000000000000000000000000000000000000000001a7ad8e3400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005f4e8fe246e26df6a55825fc5c6af245284ed8791df42e21c944c93609c175cc6c206f7000000000000000000000000000000000000000000000000000000000000001c000000000000000000000000000000000000000000000000000000000000002a00000000000000000000000000000000000000000000000000000000000000380000000000000000000000000000000000000000000000000000000000000038000000000000000000000000000000000000000000000000000000000000000a4dc1600f3000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec7000000000000000000000000af76a7c57b441501bb7a6354728e3f5e1387213900000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000000020000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb48000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb4800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000001040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000c00000000000000000000000000000000000000000000000000000000000000003000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb48000000000000000000000000dac17f958d2ee523a2206206994597c13d831ec7000000000000000000000000eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee0000000000000000000000000000000000000000000000000000000000000000869584cd00000000000000000000000086003b044f70dac0abc80ac8957305b6370893ed000000000000000000000000000000000000000000000033914ab5c85f4e73c2',
+      date: new Date('2020-09-01T16:19:01.000Z'),
+      from: '0x9b43818e253baed56e14a96d677347a06c77c6c1',
+      gasLimit: 360091,
+      gasPrice: '476000000000',
+      gasUsed: 301570,
+      hash:
+        '0x70462e739348dfab76384297b7461293800163691b28b3ba5413c07465b6b6f1',
+      index: 97,
+      nonce: '690',
+      quoteDate: new Date('2020-09-01T16:16:02.000Z'),
+      to: '0xdef1c0ded9bec7f1a1670819833240f027b25eff',
+      value: '0',
+    });
+
+    const job = { data: { eventId: '5f4e754b6609e17174cbf1d2' } };
+
+    await createTransformedERC20EventFills.fn(job, mockOptions);
+
+    const fills = await Fill.find().lean();
+
+    expect(fills).toHaveLength(1);
+  });
 });