tests: replace real currencyrate endpoints with unit tests and http error code checks

specifically coingecko was constantly hitting API rate limits causing our tests to fail
so lets unit tests all endpoints with a snapshot of real responses and only allow for http
errors in integration tests

If the actual API's change, break, or vanish we have to notice it in the logs

Changelog-None

Author: daywalker90

plugins/currencyrate-plugin/src/oracle.rs

@@ -69,58 +69,26 @@ impl Source {
         let currency = currency.to_uppercase();
         let url = self.url(&currency_lc, &currency);
 
-        let resp: Value = client
+        let response = client
             .get(&url)
             .send()
             .await
-            .map_err(|e| anyhow!("Failed to request url {url} caused by: {:?}", e.source()))?
+            .map_err(|e| anyhow!("Request failed {url}: {:?}", e.source()))?;
+
+        let status = response.status();
+
+        let resp: Value = response
             .json()
             .await
-            .map_err(|e| {
-                anyhow!(
-                    "Failed to decode response body from {url}, caused by: {:?}",
-                    e.source()
-                )
-            })?;
-
-        let reply_members = self.reply_members(&currency_lc, &currency);
+            .map_err(|e| anyhow!("Failed to parse JSON from {url}: {:?}", e.source()))?;
 
-        let mut current = &mut resp.clone();
-        for member in reply_members {
-            if let Ok(pos) = member.parse::<usize>() {
-                current = current.get_mut(pos).ok_or(anyhow!(
-                    "Positional member `{}` not found in {} response: {}",
-                    member,
-                    self.name(),
-                    resp
-                ))?;
-            } else {
-                current = current.get_mut(&member).ok_or(anyhow!(
-                    "Member `{}` not found in {} response: {}",
-                    member,
-                    self.name(),
-                    resp
-                ))?;
-            }
+        if !status.is_success() {
+            return Err(anyhow!("HTTP error {status} from {url}: body={resp}"));
         }
-        let price = match current {
-            Value::Number(number) => number
-                .as_f64()
-                .ok_or(anyhow!("Json number price could not be converted to float"))?,
-            Value::String(string) => string
-                .parse::<f64>()
-                .map_err(|e| anyhow!("Price string could not be converted to float: {e}"))?,
-            _ => return Err(anyhow!("Price is invalid json type")),
-        };
 
-        if price == 0.0 {
-            log::warn!("{} returned 0.0 as price for {}", self.name, currency);
-            return Err(anyhow!(
-                "{} returned 0.0 as price for {}",
-                self.name,
-                currency
-            ));
-        }
+        let reply_members = self.reply_members(&currency_lc, &currency);
+
+        let price = extract_price_from_response(&resp, &reply_members, self.name(), &currency)?;
 
         log::info!(
             "Fetched price in {}ms from {}: {:.2} {currency}/BTC",
@@ -133,6 +101,40 @@ impl Source {
     }
 }
 
+fn extract_price_from_response(
+    resp: &Value,
+    reply_members: &[String],
+    name: &str,
+    currency: &str,
+) -> Result<f64, anyhow::Error> {
+    let mut current = &mut resp.clone();
+    for member in reply_members {
+        if let Ok(pos) = member.parse::<usize>() {
+            current = current.get_mut(pos).ok_or(anyhow!(
+                "Positional member `{member}` not found in {name} response: {resp}"
+            ))?;
+        } else {
+            current = current.get_mut(member).ok_or(anyhow!(
+                "Member `{member}` not found in {name} response: {resp}"
+            ))?;
+        }
+    }
+    let price = match current {
+        Value::Number(number) => number
+            .as_f64()
+            .ok_or(anyhow!("Json number price could not be converted to float"))?,
+        Value::String(string) => string
+            .parse::<f64>()
+            .map_err(|e| anyhow!("Price string could not be converted to float: {e}"))?,
+        _ => return Err(anyhow!("Price is invalid json type")),
+    };
+
+    if price == 0.0 {
+        log::warn!("{name} returned 0.0 as price for {currency}");
+        return Err(anyhow!("{name} returned 0.0 as price for {currency}"));
+    }
+    Ok(price)
+}
 struct SourceHealth {
     source: Source,
     failures: u32,
@@ -535,12 +537,7 @@ impl BtcPriceOracle {
 
         // Give a helpful error if the source name is unknown entirely
         if !inner.sources.contains_key(source_name) {
-            let available = inner
-                .sources
-                .keys()
-                .cloned()
-                .collect::<Vec<_>>()
-                .join(", ");
+            let available = inner.sources.keys().cloned().collect::<Vec<_>>().join(", ");
             return Err(anyhow!(
                 "Unknown source `{source_name}`. Available sources: {available}"
             ));
@@ -551,13 +548,12 @@ impl BtcPriceOracle {
             .get(currency)
             .ok_or_else(|| anyhow!("No rates available for `{currency}`"))?;
 
-        let price_cache = currency_cache
-            .prices
-            .get(source_name)
-            .ok_or_else(|| anyhow!(
+        let price_cache = currency_cache.prices.get(source_name).ok_or_else(|| {
+            anyhow!(
                 "Source `{source_name}` has no data for `{currency}`. \
                 The source may not support this currency or is currently backing off."
-            ))?;
+            )
+        })?;
 
         if price_cache.timestamp + SERVE_TTL <= Instant::now() {
             return Err(anyhow!("Cached rate from `{source_name}` is expired"));
@@ -577,3 +573,164 @@ fn get_median(source_results: Vec<SourceResult>) -> f64 {
         f64::midpoint(prices[mid - 1], prices[mid])
     }
 }
+
+#[test]
+fn test_sources() {
+    use crate::add_default_sources;
+    use serde_json::json;
+    let mut sources = Vec::new();
+    add_default_sources(&mut sources, false);
+    let mut responses = HashMap::new();
+
+    let coingecko_price = 72732f64;
+    let coingecko_response = json!({"bitcoin": {"usd": coingecko_price}});
+    responses.insert("coingecko", (coingecko_response, coingecko_price));
+
+    let kraken_price = 72745.00000;
+    let kraken_reponse = json!({
+        "error": [],
+        "result": {
+            "XXBTZUSD": {
+                "a": ["72745.00000", "1", "1.000"],
+                "b": ["72744.90000", "3", "3.000"],
+                "c": [kraken_price, "0.00033610"],
+                "v": ["299.69911717", "640.87872182"],
+                "p": ["73231.75442", "73435.07382"],
+                "t": [17317, 39634],
+                "l": ["72613.70000", "72613.70000"],
+                "h": ["73960.00000", "74070.00000"],
+                "o": "73569.90000",
+            }
+        },
+    });
+    responses.insert("kraken", (kraken_reponse, kraken_price));
+
+    let blockchain_info_price = 72749.07;
+    let blockchain_info_reponse = json!({
+        "ARS": {
+            "15m": 1.0250388182e8,
+            "last": 1.0250388182e8,
+            "buy": 1.0250388182e8,
+            "sell": 1.0250388182e8,
+            "symbol": "ARS",
+        },
+        "AUD": {
+            "15m": 101319.81,
+            "last": 101319.81,
+            "buy": 101319.81,
+            "sell": 101319.81,
+            "symbol": "AUD",
+        },
+        "BRL": {
+            "15m": 366724.43,
+            "last": 366724.43,
+            "buy": 366724.43,
+            "sell": 366724.43,
+            "symbol": "BRL",
+        },
+        "CAD": {
+            "15m": 100512.08,
+            "last": 100512.08,
+            "buy": 100512.08,
+            "sell": 100512.08,
+            "symbol": "CAD",
+        },
+        "USD": {
+            "15m": 72749.07,
+            "last": blockchain_info_price,
+            "buy": 72749.07,
+            "sell": 72749.07,
+            "symbol": "USD",
+        },
+    });
+    responses.insert(
+        "blockchain.info",
+        (blockchain_info_reponse, blockchain_info_price),
+    );
+
+    let bitstamp_price = 72748.45;
+    let bitstamp_reponse = json!({
+        "timestamp": "1780301548",
+        "open": "73568.00",
+        "high": "74094.65",
+        "low": "72611.68",
+        "last": bitstamp_price,
+        "volume": "619.76380694",
+        "vwap": "73542.01",
+        "bid": "72748.45",
+        "ask": "72748.46",
+        "side": "1",
+        "open_24": "73769.45",
+        "percent_change_24": "-1.38",
+        "market_type": "SPOT",
+    });
+    responses.insert("bitstamp", (bitstamp_reponse, bitstamp_price));
+
+    let coindesk_price = 72751.6828660636;
+    let coindes_response = json!({
+        "Data": {
+            "BTC-USD": {
+                "TYPE": "266",
+                "MARKET": "cadli",
+                "INSTRUMENT": "BTC-USD",
+                "CCSEQ": 1323841393,
+                "VALUE": coindesk_price,
+                "VALUE_FLAG": "UP",
+                "VALUE_LAST_UPDATE_TS": 1780301570,
+                "VALUE_LAST_UPDATE_TS_NS": 94000000,
+                "LAST_UPDATE_QUANTITY": 0.105,
+                "LAST_UPDATE_QUOTE_QUANTITY": 7642.36250656015,
+                "LAST_UPDATE_VOLUME_TOP_TIER": 0,
+                "LAST_UPDATE_QUOTE_VOLUME_TOP_TIER": 0,
+                "LAST_UPDATE_VOLUME_DIRECT": 0,
+                "LAST_UPDATE_CCSEQ": 1323894738,
+                "CURRENT_HOUR_VOLUME": 1574.29740546284,
+                "CURRENT_HOUR_QUOTE_VOLUME": 114448749.088967,
+                "CURRENT_HOUR_VOLUME_TOP_TIER": 815.533688797,
+                "CURRENT_HOUR_QUOTE_VOLUME_TOP_TIER": 59271744.4993065,
+                "CURRENT_YEAR_CHANGE_PERCENTAGE": -16.8891195575271,
+                "MOVING_24_HOUR_VOLUME": 127405.003580434,
+                "MOVING_24_HOUR_QUOTE_VOLUME": 9365693546.36435,
+                "MOVING_24_HOUR_VOLUME_TOP_TIER": 60402.3558377577,
+                "MOVING_24_HOUR_QUOTE_VOLUME_TOP_TIER_DIRECT": 867716024.677109,
+                "MOVING_24_HOUR_OPEN": 73850.0780201078,
+                "MOVING_7_DAY_HIGH": 77996.1623459993,
+                "MOVING_7_DAY_LOW": 72425.5243349043,
+                "MOVING_7_DAY_TOTAL_INDEX_UPDATES": 12822192,
+                "MOVING_7_DAY_CHANGE": -4535.8043571581,
+                "MOVING_365_DAY_CHANGE_PERCENTAGE": -31.154821757090602,
+                "LIFETIME_FIRST_UPDATE_TS": 1279408140,
+                "LIFETIME_QUOTE_VOLUME_TOP_TIER_DIRECT": 4851586542024.82,
+                "LIFETIME_OPEN": 0.04951,
+                "LIFETIME_LOW": 0.01,
+                "LIFETIME_LOW_TS": 1286572500,
+                "LIFETIME_TOTAL_INDEX_UPDATES": 1329464334,
+                "LIFETIME_CHANGE": 72751.6333560636,
+                "LIFETIME_CHANGE_PERCENTAGE": 146943311.16151,
+            }
+        },
+        "Err": {},
+    });
+    responses.insert("coindesk", (coindes_response, coindesk_price));
+
+    let coinbase_price = 72760.125;
+    let coinbase_reponse =
+        json!({"data": {"amount": coinbase_price, "base": "BTC", "currency": "USD"}});
+    responses.insert("coinbase", (coinbase_reponse, coinbase_price));
+
+    let binance_price = 72715.35000000;
+    let binance_reponse = json!({"symbol": "BTCUSD", "price": binance_price});
+    responses.insert("binance", (binance_reponse, binance_price));
+
+    for source in &sources {
+        let (response, price) = responses.get(source.name()).unwrap();
+        let extracted_price = extract_price_from_response(
+            response,
+            &source.reply_members("usd", "USD"),
+            &source.name,
+            "USD",
+        )
+        .unwrap();
+        assert_eq!(extracted_price, *price, "Failed for {}", source.name());
+    }
+}