Undo unnecessary changes and fix coinank endpoint

Author: Zaczero

api/cbbiinfo_api.py

@@ -15,7 +15,6 @@ def cbbi_fetch(key: str) -> pd.DataFrame:
             'Value',
         ],
     )
-    # Convert string timestamps to int before datetime conversion to avoid FutureWarning
-    df['Date'] = pd.to_datetime(df['Date'].astype(int), unit='s').dt.tz_localize(None)
+    df['Date'] = pd.to_datetime(df['Date'], unit='s').dt.tz_localize(None)
 
     return df

api/coinsoto_api.py

@@ -4,7 +4,7 @@
 
 
 def cs_fetch(path: str, data_selector: str, col_name: str) -> pd.DataFrame:
-    response = HTTP.get(f'https://coinank.com/indicatorapi/{path}')
+    response = HTTP.get(f'https://api.coinank.com/indicatorapi/{path}')
     response.raise_for_status()
     data = response.json()['data']
 
@@ -15,12 +15,10 @@ def cs_fetch(path: str, data_selector: str, col_name: str) -> pd.DataFrame:
     data_y = data[data_selector]
     assert len(data_x) == len(data_y), f'{len(data_x)=} != {len(data_y)=}'
 
-    df = pd.DataFrame(
-        {
-            'Date': data_x[: len(data_y)],
-            col_name: data_y,
-        }
-    )
+    df = pd.DataFrame({
+        'Date': data_x[: len(data_y)],
+        col_name: data_y,
+    })
 
     df['Date'] = pd.to_datetime(df['Date'], unit='ms').dt.tz_localize(None)
 

fetch_bitcoin_data.py

@@ -1,102 +1,106 @@
+from itertools import count, pairwise
+
 import numpy as np
 import pandas as pd
 from filecache import filecache
 
-from utils import HTTP, http_get_with_retry, mark_days_since, mark_highs_lows
+from utils import HTTP, mark_days_since, mark_highs_lows
+
+HALVING_INTERVAL = 210_000
+GENESIS_BLOCK_REWARD = 50.0
+BLOCKS_PER_DAY = 144
+
+
+def fetch_block_halving():
+    """
+    Fetch Bitcoin halving data by probing raw blocks at halving heights.
+    """
+    halving_data: list[tuple[pd.Timestamp, int, float]] = []
+
+    for halving_index in count():
+        block_height = halving_index * HALVING_INTERVAL
+        block_reward = GENESIS_BLOCK_REWARD / (2**halving_index)
+
+        response = HTTP.get(f'https://blockchain.info/rawblock/{block_height}')
+        if response.status_code == 404:
+            break
 
-# Known Bitcoin halving dates and block heights for accurate calculations
-HALVING_DATA = [
-    # (date, block_height, block_reward)
-    ('2009-01-03', 0, 50.0),           # Genesis block
-    ('2012-11-28', 210000, 25.0),      # 1st halving
-    ('2016-07-09', 420000, 12.5),      # 2nd halving
-    ('2020-05-11', 630000, 6.25),      # 3rd halving
-    ('2024-04-20', 840000, 3.125),     # 4th halving
-]
+        response.raise_for_status()
+        block_time = response.json()['time']
+        block_date = pd.to_datetime(block_time, unit='s').tz_localize(None).floor('d')
+        halving_data.append((block_date, block_height, block_reward))
+
+    return halving_data
 
 
 def fetch_blockchain_data() -> pd.DataFrame:
     """
     Fetches historical Bitcoin blockchain data from Blockchain.com API.
     Uses miners-revenue chart for USD mining revenue.
     Block heights and BTC generation are calculated from halving schedule.
-    
+
     Returns:
-        DataFrame with Date, TotalBlocks, MinBlockID, MaxBlockID, 
+        DataFrame with Date, TotalBlocks, MinBlockID, MaxBlockID,
         TotalGeneration, TotalGenerationUSD columns.
     """
-    # Fetch mining revenue from Blockchain.com (free, reliable)
-    response = http_get_with_retry(
+    halving_data = fetch_block_halving()
+
+    # Fetch mining revenue from Blockchain.com
+    response = HTTP.get(
         'https://api.blockchain.info/charts/miners-revenue',
         params={
             'timespan': 'all',
             'format': 'json',
             'sampled': 'false',
         },
     )
-    revenue_data = response.json()
-    
+
     # Create DataFrame from mining revenue data
-    df = pd.DataFrame(revenue_data['values'])
+    response.raise_for_status()
+    df = pd.DataFrame(response.json()['values'])
     df.columns = ['DateTimestamp', 'TotalGenerationUSD']
     df['Date'] = pd.to_datetime(df['DateTimestamp'], unit='s').dt.floor('d')
-    
-    # Calculate block heights based on known halving dates
-    # Average ~144 blocks per day (one block every 10 minutes)
-    genesis_date = pd.Timestamp('2009-01-03')
-    
-    # Create halving schedule DataFrame for interpolation
-    halving_df = pd.DataFrame(HALVING_DATA, columns=['Date', 'BlockHeight', 'BlockReward'])
-    halving_df['Date'] = pd.to_datetime(halving_df['Date'])
-    
+
     # Calculate approximate block height for each day using linear interpolation
     # between known halving points
-    def estimate_block_height(date):
-        date = pd.Timestamp(date)
-        if date < genesis_date:
-            return 0
-        
+    def estimate_block_height(date: pd.Timestamp):
         # Find the halving period this date falls into
-        for i in range(len(HALVING_DATA) - 1):
-            start_date = pd.Timestamp(HALVING_DATA[i][0])
-            end_date = pd.Timestamp(HALVING_DATA[i + 1][0])
-            start_height = HALVING_DATA[i][1]
-            end_height = HALVING_DATA[i + 1][1]
-            
+        for (start_date, start_height, _), (end_date, end_height, _) in pairwise(halving_data):
             if start_date <= date < end_date:
                 # Linear interpolation within this halving period
                 total_days = (end_date - start_date).days
                 days_elapsed = (date - start_date).days
                 height = start_height + (end_height - start_height) * days_elapsed / total_days
                 return int(height)
-        
-        # After the last known halving, extrapolate at ~144 blocks/day
-        last_date = pd.Timestamp(HALVING_DATA[-1][0])
-        last_height = HALVING_DATA[-1][1]
+
+        # After the last known halving, extrapolate
+        (last_date, last_height, _) = halving_data[-1]
         days_since = (date - last_date).days
-        return int(last_height + days_since * 144)
-    
+        return int(last_height + days_since * BLOCKS_PER_DAY)
+
     def get_block_reward(block_height):
         """Get block reward for a given block height."""
-        halving_interval = 210000
-        halvings = block_height // halving_interval
-        return 50.0 / (2 ** halvings)
-    
+        halvings = block_height // HALVING_INTERVAL
+        return GENESIS_BLOCK_REWARD / (2**halvings)
+
     # Calculate block data for each day
     df['MaxBlockID'] = df['Date'].apply(estimate_block_height)
-    df['MinBlockID'] = df['MaxBlockID'].shift(1).fillna(0).astype(int)
+    df['MinBlockID'] = df['MaxBlockID'].shift(1, fill_value=0)
     df['TotalBlocks'] = df['MaxBlockID'] - df['MinBlockID']
-    df['TotalBlocks'] = df['TotalBlocks'].clip(lower=1)  # Ensure at least 1 block
-    
+
     # Calculate BTC generation based on block reward
-    # Store in satoshis (multiply by 1e8) to match original Blockchair format
     df['BlockReward'] = df['MaxBlockID'].apply(get_block_reward)
     df['TotalGeneration'] = df['TotalBlocks'] * df['BlockReward'] * 1e8  # Convert to satoshis
-    
+
     # Select and order columns to match original format
     df = df[['Date', 'TotalBlocks', 'MinBlockID', 'MaxBlockID', 'TotalGeneration', 'TotalGenerationUSD']]
     df = df.sort_values('Date').reset_index(drop=True)
-    
+
+    # Add halving markers
+    df['Halving'] = 0
+    for _, block_height, _ in halving_data[1:]:
+        df.loc[(df['MinBlockID'] < block_height) & (df['MaxBlockID'] >= block_height), 'Halving'] = 1
+
     return df
 
 
@@ -111,7 +115,6 @@ def fetch_bitcoin_data() -> pd.DataFrame:
     """
     print('📈 Requesting historical Bitcoin data…')
 
-    # Use Blockchain.com API instead of Blockchair (which is blocked)
     df = fetch_blockchain_data()
 
     df['Date'] = pd.to_datetime(df['Date'])
@@ -132,7 +135,6 @@ def fetch_bitcoin_data() -> pd.DataFrame:
     df.reset_index(drop=True, inplace=True)
 
     df = fix_current_day_data(df)
-    df = add_block_halving_data(df)
     df = mark_highs_lows(df, 'Price', False, round(365 * 2), 180)
 
     # move 2021' peak to the first price peak
@@ -144,24 +146,23 @@ def fetch_bitcoin_data() -> pd.DataFrame:
 
 
 def fetch_price_data() -> pd.DataFrame:
-    response = http_get_with_retry(
+    response = HTTP.get(
         'https://api.coinmarketcap.com/data-api/v3/cryptocurrency/detail/chart',
         params={
             'id': 1,
             'range': 'ALL',
         },
     )
 
+    response.raise_for_status()
     response_json = response.json()
     response_x = [float(k) for k in response_json['data']['points']]
     response_y = [value['v'][0] for value in response_json['data']['points'].values()]
 
-    df = pd.DataFrame(
-        {
-            'Date': response_x,
-            'Price': response_y,
-        }
-    )
+    df = pd.DataFrame({
+        'Date': response_x,
+        'Price': response_y,
+    })
     df['Date'] = pd.to_datetime(df['Date'], unit='s').dt.tz_localize(None).dt.floor('d')
     df.sort_values(by='Date', inplace=True)
     df.drop_duplicates('Date', keep='last', inplace=True)
@@ -172,45 +173,10 @@ def fetch_price_data() -> pd.DataFrame:
 def fix_current_day_data(df: pd.DataFrame) -> pd.DataFrame:
     row = df.iloc[-1].copy()
 
-    target_total_blocks = 24 * 6
-    target_scale = target_total_blocks / row['TotalBlocks']
+    target_scale = BLOCKS_PER_DAY / row['TotalBlocks']
 
     for col_name in ['TotalBlocks', 'TotalGeneration', 'TotalGenerationUSD']:
         row[col_name] *= target_scale
 
     df.iloc[-1] = row
     return df
-
-
-def add_block_halving_data(df: pd.DataFrame) -> pd.DataFrame:
-    reward_halving_every = 210000
-    current_block_halving_id = reward_halving_every
-    current_block_production = 50
-    df['Halving'] = 0
-    df['NextHalvingBlock'] = current_block_halving_id
-
-    while True:
-        df.loc[
-            (current_block_halving_id - reward_halving_every) <= df['MaxBlockID'],
-            'BlockGeneration',
-        ] = current_block_production
-
-        block_halving_rows = df[
-            (df['MinBlockID'] <= current_block_halving_id) & (df['MaxBlockID'] >= current_block_halving_id)
-        ]
-
-        if len(block_halving_rows) == 0:
-            break
-
-        # Take the first matching row if multiple match
-        block_halving_row = block_halving_rows.iloc[0]
-        row_index = block_halving_rows.index[0]
-
-        current_block_halving_id += reward_halving_every
-        current_block_production /= 2
-        df.loc[row_index, 'Halving'] = 1
-        df.loc[df.index > row_index, 'NextHalvingBlock'] = current_block_halving_id
-
-    df['DaysToHalving'] = pd.to_timedelta((df['NextHalvingBlock'] - df['MaxBlockID']) / (24 * 6), unit='D')
-    df['NextHalvingDate'] = df['Date'] + df['DaysToHalving']
-    return df

metrics/base_metric.py

@@ -1,3 +1,4 @@
+import traceback
 from abc import ABC, abstractmethod
 
 import pandas as pd
@@ -33,7 +34,7 @@ async def calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series:
         try:
             return self._calculate(df, ax)
         except Exception as ex:
-            # Silently fall back - no traceback printed to keep logs clean
+            traceback.print_exc()
             await send_error_notification(ex)
 
             print(fg.black + bg.yellow + f' Requesting fallback values for {self.name} (from CBBI.info) ' + rs.all)

metrics/pi_cycle.py

@@ -63,4 +63,8 @@ def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series:
         sns.lineplot(data=df, x='Date', y='PiCycleIndexNoNa', ax=ax[0])
         add_common_markers(df, ax[0])
 
+        sns.lineplot(data=df, x='Date', y='PiCycleDiff', ax=ax[1])
+        sns.lineplot(data=df, x='Date', y='PiCycleDiffThreshold', ax=ax[1], linestyle='--')
+        add_common_markers(df, ax[1], price_line=False)
+
         return df['PiCycleIndex']

metrics/puell_multiple.py

@@ -21,8 +21,7 @@ def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series:
         # Calculate Puell Multiple locally from mining revenue data
         # Puell = daily_mining_revenue / 365-day_MA_of_mining_revenue
         # TotalGenerationUSD contains daily mining revenue in USD from Blockchain.com
-        df['MiningRevenue365MA'] = df['TotalGenerationUSD'].rolling(window=365, min_periods=1).mean()
-        df['Puell'] = df['TotalGenerationUSD'] / df['MiningRevenue365MA']
+        df['Puell'] = df['TotalGenerationUSD'] / df['TotalGenerationUSD'].rolling(window=365, min_periods=1).mean()
         df['Puell'] = df['Puell'].ffill()
         df['PuellLog'] = np.log(df['Puell'])
 

metrics/rhodl_ratio.py

@@ -1,3 +1,5 @@
+import traceback
+
 import numpy as np
 import pandas as pd
 import seaborn as sns
@@ -28,7 +30,7 @@ def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series:
                 col_name='RHODL',
             )
         except Exception:
-            # Silently try GlassNode fallback - no traceback printed
+            traceback.print_exc()
             print(fg.black + bg.yellow + f' Requesting fallback values for {self.name} (from GlassNode) ' + rs.all)
 
             remote_df = gn_fetch(url_selector='rhodl_ratio', col_name='RHODL', a='BTC')