Optimize rates engine

- Round robin query all rates servers
- Increase batch size and query frequency
- Do not write unchanged docs

Author: paullinator

src/ratesEngine.ts

@@ -15,8 +15,15 @@ import { datelog, safeParseFloat, standardizeNames } from './util'
 import { isFiatCurrency } from './util/fiatCurrency'
 
 const nanoDb = nano(config.couchDbFullpath)
-const QUERY_FREQ_MS = 3000
-const QUERY_LIMIT = 10
+const QUERY_FREQ_MS = 2000
+const QUERY_LIMIT = 20
+const RATES_SERVERS = [
+  'https://rates1.edge.app',
+  'https://rates2.edge.app',
+  'https://rates3.edge.app',
+  'https://rates4.edge.app'
+]
+
 const snooze: Function = async (ms: number) =>
   await new Promise((resolve: Function) => setTimeout(resolve, ms))
 
@@ -178,7 +185,9 @@ async function updateTxValuesV3(transaction: DbTx): Promise<void> {
     return
   }
 
-  const ratesResponse = await fetch('https://rates3.edge.app/v3/rates', {
+  const server = RATES_SERVERS[Math.floor(Math.random() * RATES_SERVERS.length)]
+  datelog(`Getting v3 rates from ${server}`)
+  const ratesResponse = await fetch(`${server}/v3/rates`, {
     method: 'POST',
     headers: {
       'Content-Type': 'application/json'
@@ -205,6 +214,7 @@ async function updateTxValuesV3(transaction: DbTx): Promise<void> {
   const depositRate = depositRateObf?.rate
   const payoutRate = payoutRateObf?.rate
 
+  let changed = false
   // Calculate and fill out payoutAmount if it is zero
   if (payoutAmount === 0) {
     if (depositRate == null) {
@@ -220,6 +230,7 @@ async function updateTxValuesV3(transaction: DbTx): Promise<void> {
     }
     if (depositRate != null && payoutRate != null) {
       transaction.payoutAmount = (depositAmount * depositRate) / payoutRate
+      changed = true
     }
   }
 
@@ -229,6 +240,7 @@ async function updateTxValuesV3(transaction: DbTx): Promise<void> {
   if (transaction.usdValue == null || transaction.usdValue <= 0) {
     if (depositRate != null) {
       transaction.usdValue = depositAmount * depositRate
+      changed = true
       datelog(
         `V3 SUCCESS id:${t._id} ${t.isoDate.slice(0, 10)} deposit:${
           t.depositCurrency
@@ -238,6 +250,7 @@ async function updateTxValuesV3(transaction: DbTx): Promise<void> {
       )
     } else if (payoutRate != null) {
       transaction.usdValue = transaction.payoutAmount * payoutRate
+      changed = true
       datelog(
         `V3 SUCCESS id:${t._id} ${t.isoDate.slice(0, 10)} payout:${
           t.payoutCurrency
@@ -247,6 +260,14 @@ async function updateTxValuesV3(transaction: DbTx): Promise<void> {
       )
     }
   }
+  if (!changed) {
+    datelog(
+      `V3 NO CHANGE id:${t._id} ${t.isoDate.slice(0, 10)} ${
+        t.depositCurrency
+      } ${t.payoutCurrency}`
+    )
+    transaction._id = undefined
+  }
 }
 
 async function updateTxValues(
@@ -394,7 +415,9 @@ async function getExchangeRate(
   currencyA = isFiatCurrency(currencyA) ? `iso:${currencyA}` : currencyA
   currencyB = isFiatCurrency(currencyB) ? `iso:${currencyB}` : currencyB
 
-  const url = `https://rates2.edge.app/v2/exchangeRate?currency_pair=${currencyA}_${currencyB}&date=${hourDate}`
+  const server = RATES_SERVERS[Math.floor(Math.random() * RATES_SERVERS.length)]
+  const url = `${server}/v2/exchangeRate?currency_pair=${currencyA}_${currencyB}&date=${hourDate}`
+  datelog(`Getting v2 exchange rate from ${server}`)
   try {
     const result = await fetch(url, { method: 'GET' })
     if (!result.ok) {