diff --git a/app/auth.py b/app/auth.py index 9794ffc..7665b20 100644 --- a/app/auth.py +++ b/app/auth.py @@ -5,9 +5,11 @@ from __future__ import annotations +import hmac import logging import os import secrets +import time from pathlib import Path from typing import Any @@ -75,6 +77,10 @@ def _resolve_secret_key() -> str: _serializer = URLSafeTimedSerializer(SECRET_KEY) +# Global session epoch: tokens issued before this timestamp are rejected. +# Bump via env var to mass-invalidate all sessions (e.g. after a credential leak). +_SESSION_EPOCH = float(os.getenv("CASHPILOT_SESSION_EPOCH", "0")) + def hash_password(password: str) -> str: # bcrypt enforces a 72-byte limit; truncate UTF-8 bytes (not characters) @@ -88,12 +94,16 @@ def verify_password(password: str, hashed: str) -> bool: def create_session_token(user_id: int, username: str, role: str) -> str: - return _serializer.dumps({"uid": user_id, "u": username, "r": role}) + return _serializer.dumps({"uid": user_id, "u": username, "r": role, "iat": time.time()}) def decode_session_token(token: str) -> dict[str, Any] | None: try: - return _serializer.loads(token, max_age=SESSION_MAX_AGE) + data = _serializer.loads(token, max_age=SESSION_MAX_AGE) + # Reject tokens issued before session epoch (for mass invalidation) + if data.get("iat", 0) < _SESSION_EPOCH: + return None + return data except (BadSignature, Exception): return None @@ -108,10 +118,10 @@ def get_current_user(request: Request) -> dict[str, Any] | None: auth_header = request.headers.get("Authorization", "") if auth_header: admin_key = os.getenv("CASHPILOT_ADMIN_API_KEY", "") - if admin_key and auth_header == f"Bearer {admin_key}": + if admin_key and hmac.compare_digest(auth_header.encode(), f"Bearer {admin_key}".encode()): return {"uid": 0, "u": "api", "r": "owner"} resolved_fleet_key = _fleet_key_mod.resolve_fleet_key() - if resolved_fleet_key and auth_header == f"Bearer {resolved_fleet_key}": + if resolved_fleet_key and hmac.compare_digest(auth_header.encode(), f"Bearer {resolved_fleet_key}".encode()): return {"uid": 0, "u": "fleet", "r": "fleet"} # Fall back to session cookie diff --git a/app/collectors/__init__.py b/app/collectors/__init__.py index 438029e..1f470fd 100644 --- a/app/collectors/__init__.py +++ b/app/collectors/__init__.py @@ -63,22 +63,30 @@ "salad": ["auth_cookie"], } +_cached_collectors: dict[str, BaseCollector] = {} +_cached_kwargs: dict[str, dict[str, str]] = {} +_stale: list[BaseCollector] = [] + + +async def _close_stale() -> None: + """Close collectors evicted from cache due to config changes.""" + global _stale + for c in _stale: + await c.close() + _stale = [] + def make_collectors( deployments: list[dict[str, Any]], config: dict[str, str], ) -> list[BaseCollector]: - """Create collector instances for all deployed services that have collectors. + """Create or retrieve cached collector instances for deployed services. - Args: - deployments: List of deployment dicts (must have 'slug' key). - config: User config dict. Collector credentials are stored as - ``{slug}_email``, ``{slug}_password``, etc. - - Returns: - List of ready-to-use collector instances. + Reuses a cached instance when the resolved kwargs for a slug match + the previous invocation. Evicts stale instances when config changes. """ collectors: list[BaseCollector] = [] + active_slugs: set[str] = set() for dep in deployments: slug = dep.get("slug", "") @@ -89,7 +97,6 @@ def make_collectors( arg_keys = _COLLECTOR_ARGS.get(slug, []) # Resolve constructor kwargs from config - # Args prefixed with ? are optional kwargs: dict[str, str] = {} missing: list[str] = [] for arg in arg_keys: @@ -110,18 +117,50 @@ def make_collectors( ) continue + active_slugs.add(slug) + + # Reuse cached instance if kwargs unchanged + if slug in _cached_collectors and _cached_kwargs.get(slug) == kwargs: + collectors.append(_cached_collectors[slug]) + logger.debug("Reusing cached collector for %s", slug) + continue + + # Config changed or new slug — evict old instance + if slug in _cached_collectors: + _stale.append(_cached_collectors[slug]) + try: - collectors.append(cls(**kwargs)) + instance = cls(**kwargs) + _cached_collectors[slug] = instance + _cached_kwargs[slug] = kwargs + collectors.append(instance) logger.debug("Created collector for %s", slug) except Exception as exc: logger.error("Failed to create collector for %s: %s", slug, exc) + # Evict collectors for slugs no longer deployed + for slug in list(_cached_collectors.keys()): + if slug not in active_slugs: + _stale.append(_cached_collectors.pop(slug)) + _cached_kwargs.pop(slug, None) + return collectors +async def close_all_collectors() -> None: + """Close all cached collector HTTP clients and clear the cache.""" + global _cached_collectors, _cached_kwargs + for collector in _cached_collectors.values(): + await collector.close() + _cached_collectors = {} + _cached_kwargs = {} + await _close_stale() + + __all__ = [ "BaseCollector", "EarningsResult", "COLLECTOR_MAP", "make_collectors", + "close_all_collectors", ] diff --git a/app/collectors/base.py b/app/collectors/base.py index 12b8a6c..b2a0609 100644 --- a/app/collectors/base.py +++ b/app/collectors/base.py @@ -2,7 +2,14 @@ from __future__ import annotations +import asyncio +from collections.abc import Awaitable, Callable from dataclasses import dataclass +from typing import Any, TypeVar + +import httpx + +T = TypeVar("T") @dataclass @@ -12,7 +19,6 @@ class EarningsResult: platform: str balance: float currency: str = "USD" - bytes_uploaded: int = 0 error: str | None = None @@ -24,5 +30,39 @@ class BaseCollector: platform: str = "" + def __init__(self) -> None: + self._client: httpx.AsyncClient | None = None + + def _get_client(self, **kwargs: Any) -> httpx.AsyncClient: + """Return a reusable httpx client, creating one if needed.""" + if self._client is None or self._client.is_closed: + defaults: dict[str, Any] = {"timeout": 30} + defaults.update(kwargs) + self._client = httpx.AsyncClient(**defaults) + return self._client + + async def close(self) -> None: + """Close the underlying HTTP client. Safe to call multiple times.""" + if self._client is not None and not self._client.is_closed: + await self._client.aclose() + self._client = None + + async def _retry( + self, + coro_fn: Callable[[], Awaitable[T]], + max_retries: int = 2, + backoff: float = 1.0, + ) -> T: + """Retry a coroutine on transient network failures.""" + last_exc: BaseException | None = None + for attempt in range(max_retries + 1): + try: + return await coro_fn() + except (httpx.TimeoutException, httpx.NetworkError) as exc: + last_exc = exc + if attempt < max_retries: + await asyncio.sleep(backoff * (2**attempt)) + raise last_exc # type: ignore[misc] + async def collect(self) -> EarningsResult: raise NotImplementedError diff --git a/app/collectors/bitping.py b/app/collectors/bitping.py index adaa397..97acfc9 100644 --- a/app/collectors/bitping.py +++ b/app/collectors/bitping.py @@ -23,6 +23,7 @@ class BitpingCollector(BaseCollector): platform = "bitping" def __init__(self, email: str, password: str) -> None: + super().__init__() self.email = email self.password = password self._token: str | None = None @@ -51,36 +52,37 @@ async def _authenticate(self, client: httpx.AsyncClient) -> str: async def collect(self) -> EarningsResult: """Fetch current Bitping earnings.""" try: - async with httpx.AsyncClient(timeout=30) as client: - if not self._token: - self._token = await self._authenticate(client) + client = self._get_client(timeout=30) + if not self._token: + self._token = await self._authenticate(client) - headers = {"Authorization": f"Bearer {self._token}"} - resp = await client.get( + headers = {"Authorization": f"Bearer {self._token}"} + + async def _fetch_balance() -> httpx.Response: + return await client.get( f"{API_BASE}/api/v2/payouts/earnings", headers=headers, ) - if resp.status_code == 401: - self._token = await self._authenticate(client) - headers = {"Authorization": f"Bearer {self._token}"} - resp = await client.get( - f"{API_BASE}/api/v2/payouts/earnings", - headers=headers, - ) + resp = await self._retry(_fetch_balance) - resp.raise_for_status() - data = resp.json() + if resp.status_code == 401: + self._token = await self._authenticate(client) + headers = {"Authorization": f"Bearer {self._token}"} + resp = await self._retry(_fetch_balance) - balance = float(data.get("usdEarnings", 0)) + resp.raise_for_status() + data = resp.json() - return EarningsResult( - platform=self.platform, - balance=round(balance, 4), - currency="USD", - ) + balance = float(data.get("usdEarnings", 0)) + + return EarningsResult( + platform=self.platform, + balance=round(balance, 4), + currency="USD", + ) except Exception as exc: - logger.error("Bitping collection failed: %s", exc) + logger.error("Bitping collection failed: %s", exc, exc_info=True) return EarningsResult( platform=self.platform, balance=0.0, diff --git a/app/collectors/bytelixir.py b/app/collectors/bytelixir.py index 232146e..1e71438 100644 --- a/app/collectors/bytelixir.py +++ b/app/collectors/bytelixir.py @@ -21,6 +21,7 @@ import logging import re +from typing import Any from urllib.parse import unquote import httpx @@ -46,6 +47,7 @@ def __init__( remember_web: str = "", xsrf_token: str = "", ) -> None: + super().__init__() self.session_cookie = unquote(session_cookie) self.remember_web = unquote(remember_web) if remember_web else "" self.xsrf_token = unquote(xsrf_token) if xsrf_token else "" @@ -54,31 +56,33 @@ def __init__( # Internal helpers # ------------------------------------------------------------------ - def _make_client(self) -> httpx.AsyncClient: - """Build an httpx client with all session cookies pre-set.""" - cookies = httpx.Cookies() - cookies.set( - "bytelixir_session", - self.session_cookie, - domain="dash.bytelixir.com", - ) - if self.remember_web: + def _get_client(self, **kwargs: Any) -> httpx.AsyncClient: + """Return a reusable httpx client with all session cookies pre-set.""" + if self._client is None or self._client.is_closed: + cookies = httpx.Cookies() cookies.set( - self._REMEMBER_COOKIE, - self.remember_web, + "bytelixir_session", + self.session_cookie, domain="dash.bytelixir.com", ) - if self.xsrf_token: - cookies.set( - "XSRF-TOKEN", - self.xsrf_token, - domain="dash.bytelixir.com", + if self.remember_web: + cookies.set( + self._REMEMBER_COOKIE, + self.remember_web, + domain="dash.bytelixir.com", + ) + if self.xsrf_token: + cookies.set( + "XSRF-TOKEN", + self.xsrf_token, + domain="dash.bytelixir.com", + ) + self._client = httpx.AsyncClient( + timeout=30, + cookies=cookies, + follow_redirects=True, ) - return httpx.AsyncClient( - timeout=30, - cookies=cookies, - follow_redirects=True, - ) + return self._client @staticmethod def _browser_headers(*, accept: str = "text/html") -> dict[str, str]: @@ -135,39 +139,45 @@ async def collect(self) -> EarningsResult: Fallback: ``/api/v1/user`` JSON endpoint. """ try: - async with self._make_client() as client: - # --- 1. Try scraping the dashboard HTML --------------- - # Use the root URL — when authenticated it redirects to - # the localised dashboard (e.g. /en). When not - # authenticated it redirects to /login. - html_resp = await client.get( + client = self._get_client() + + # --- 1. Try scraping the dashboard HTML --------------- + # Use the root URL — when authenticated it redirects to + # the localised dashboard (e.g. /en). When not + # authenticated it redirects to /login. + async def _fetch_html() -> httpx.Response: + return await client.get( f"{API_BASE}/", headers=self._browser_headers(), ) - # A 302 to /login means the session cookie is expired. - if "login" in str(html_resp.url): + html_resp = await self._retry(_fetch_html) + + # A redirect away from a dashboard path means session expired. + final_path = html_resp.url.path + if not final_path.startswith(("/en", "/es", "/dashboard", "/home")): + return EarningsResult( + platform=self.platform, + balance=0.0, + error="Session expired — refresh bytelixir_session cookie in Settings", + ) + + if html_resp.status_code == 200: + scraped = self._parse_balance_from_html(html_resp.text) + if scraped is not None: + logger.info( + "Bytelixir balance from HTML scrape: %s", + scraped, + ) return EarningsResult( platform=self.platform, - balance=0.0, - error=("Session expired — refresh bytelixir_session cookie in Settings"), + balance=round(scraped, 4), + currency="USD", ) - if html_resp.status_code == 200: - scraped = self._parse_balance_from_html(html_resp.text) - if scraped is not None: - logger.info( - "Bytelixir balance from HTML scrape: %s", - scraped, - ) - return EarningsResult( - platform=self.platform, - balance=round(scraped, 4), - currency="USD", - ) - - # --- 2. Fallback: JSON API ---------------------------- - api_resp = await client.get( + # --- 2. Fallback: JSON API ---------------------------- + async def _fetch_api() -> httpx.Response: + return await client.get( f"{API_BASE}/api/v1/user", headers=self._browser_headers(accept="application/json") | { @@ -177,31 +187,33 @@ async def collect(self) -> EarningsResult: }, ) - if api_resp.status_code == 401: - return EarningsResult( - platform=self.platform, - balance=0.0, - error=("Session expired — refresh bytelixir_session cookie in Settings"), - ) - - api_resp.raise_for_status() - data = api_resp.json() - - # Response shape: {"data": {"balance": "0.0000000000", ...}} - # NOTE: /api/v1/user returns *withdrawable* balance only, not - # total earned. Flag this so the user knows it's approximate. - user_data = data.get("data", {}) - balance_str = user_data.get("balance", "0") - balance = float(balance_str) + api_resp = await self._retry(_fetch_api) + if api_resp.status_code == 401: return EarningsResult( platform=self.platform, - balance=round(balance, 4), - currency="USD", - error="Withdrawable balance only (HTML scrape failed, using API fallback)", + balance=0.0, + error="Session expired — refresh bytelixir_session cookie in Settings", ) + + api_resp.raise_for_status() + data = api_resp.json() + + # Response shape: {"data": {"balance": "0.0000000000", ...}} + # NOTE: /api/v1/user returns *withdrawable* balance only, not + # total earned. Flag this so the user knows it's approximate. + user_data = data.get("data", {}) + balance_str = user_data.get("balance", "0") + balance = float(balance_str) + + return EarningsResult( + platform=self.platform, + balance=round(balance, 4), + currency="USD", + error="Withdrawable balance only (HTML scrape failed, using API fallback)", + ) except Exception as exc: - logger.error("Bytelixir collection failed: %s", exc) + logger.error("Bytelixir collection failed: %s", exc, exc_info=True) return EarningsResult( platform=self.platform, balance=0.0, diff --git a/app/collectors/earnapp.py b/app/collectors/earnapp.py index fdd27ac..700f12a 100644 --- a/app/collectors/earnapp.py +++ b/app/collectors/earnapp.py @@ -8,22 +8,23 @@ import logging -import httpx +import httpx # noqa: F401 — needed for test patching from app.collectors.base import BaseCollector, EarningsResult logger = logging.getLogger(__name__) API_BASE = "https://earnapp.com/dashboard/api" -API_PARAMS = {"appid": "earnapp", "version": "1.613.719"} class EarnAppCollector(BaseCollector): """Collect earnings from EarnApp's dashboard API.""" platform = "earnapp" + _API_VERSION = "1.627.783" def __init__(self, oauth_token: str, brd_sess_id: str = "") -> None: + super().__init__() self.oauth_token = oauth_token self.brd_sess_id = brd_sess_id @@ -38,57 +39,67 @@ async def collect(self) -> EarningsResult: if self.brd_sess_id: cookies["brd_sess_id"] = self.brd_sess_id - async with httpx.AsyncClient(timeout=30, cookies=cookies) as client: - # Step 1: Rotate XSRF token - await client.get( - f"{API_BASE}/sec/rotate_xsrf", - params=API_PARAMS, - ) - xsrf_token = "" - for cookie_name, cookie_value in client.cookies.items(): - if cookie_name == "xsrf-token": - xsrf_token = cookie_value - break - - # Step 2: Fetch balance - headers = { - "X-Requested-With": "XMLHttpRequest", - } - if xsrf_token: - headers["xsrf-token"] = xsrf_token + api_params = {"appid": "earnapp", "version": self._API_VERSION} + client = self._get_client(timeout=30, cookies=cookies) + + # Step 1: Rotate XSRF token + await client.get( + f"{API_BASE}/sec/rotate_xsrf", + params=api_params, + ) + xsrf_token = "" + for cookie_name, cookie_value in client.cookies.items(): + if cookie_name == "xsrf-token": + xsrf_token = cookie_value + break + + # Step 2: Fetch balance + headers = { + "X-Requested-With": "XMLHttpRequest", + } + if xsrf_token: + headers["xsrf-token"] = xsrf_token + async def _fetch_balance(): resp = await client.get( f"{API_BASE}/money", headers=headers, - params=API_PARAMS, + params=api_params, ) if resp.status_code == 403: - return EarningsResult( - platform=self.platform, - balance=0.0, - error="Authentication failed — check OAuth token and session cookie", - ) + return resp resp.raise_for_status() - data = resp.json() + return resp - if "error" in data: - return EarningsResult( - platform=self.platform, - balance=0.0, - error=data["error"], - ) + resp = await self._retry(_fetch_balance) - balance = float(data.get("balance", 0)) + if resp.status_code == 403: + return EarningsResult( + platform=self.platform, + balance=0.0, + error="Authentication failed — check OAuth token and session cookie", + ) + data = resp.json() + + if "error" in data: return EarningsResult( platform=self.platform, - balance=round(balance, 4), - currency="USD", + balance=0.0, + error=data["error"], ) + + balance = float(data.get("balance", 0)) + + return EarningsResult( + platform=self.platform, + balance=round(balance, 4), + currency="USD", + ) except Exception as exc: - logger.error("EarnApp collection failed: %s", exc) + logger.error("EarnApp collection failed: %s", exc, exc_info=True) return EarningsResult( platform=self.platform, balance=0.0, diff --git a/app/collectors/earnfm.py b/app/collectors/earnfm.py index 11d3a91..95204a8 100644 --- a/app/collectors/earnfm.py +++ b/app/collectors/earnfm.py @@ -30,6 +30,7 @@ class EarnFMCollector(BaseCollector): platform = "earnfm" def __init__(self, email: str, password: str) -> None: + super().__init__() self.email = email self.password = password self._access_token: str | None = None @@ -76,36 +77,37 @@ async def _refresh(self, client: httpx.AsyncClient) -> str: async def collect(self) -> EarningsResult: """Fetch current Earn.fm balance.""" try: - async with httpx.AsyncClient(timeout=30) as client: - if not self._access_token: - await self._authenticate(client) + client = self._get_client(timeout=30) + if not self._access_token: + await self._authenticate(client) - headers = {"X-API-Key": self._access_token} - resp = await client.get( + headers = {"X-API-Key": self._access_token} + + async def _fetch_balance() -> httpx.Response: + return await client.get( f"{API_BASE}/harvester/view_balance", headers=headers, ) - if resp.status_code == 401: - await self._refresh(client) - headers = {"X-API-Key": self._access_token} - resp = await client.get( - f"{API_BASE}/harvester/view_balance", - headers=headers, - ) + resp = await self._retry(_fetch_balance) - resp.raise_for_status() - data = resp.json() + if resp.status_code == 401: + await self._refresh(client) + headers = {"X-API-Key": self._access_token} + resp = await self._retry(_fetch_balance) - balance = float(data.get("data", {}).get("totalBalance", 0)) + resp.raise_for_status() + data = resp.json() - return EarningsResult( - platform=self.platform, - balance=round(balance, 4), - currency="USD", - ) + balance = float(data.get("data", {}).get("totalBalance", 0)) + + return EarningsResult( + platform=self.platform, + balance=round(balance, 4), + currency="USD", + ) except Exception as exc: - logger.error("EarnFM collection failed: %s", exc) + logger.error("EarnFM collection failed: %s", exc, exc_info=True) return EarningsResult( platform=self.platform, balance=0.0, diff --git a/app/collectors/grass.py b/app/collectors/grass.py index 7e41ef4..dfea661 100644 --- a/app/collectors/grass.py +++ b/app/collectors/grass.py @@ -25,6 +25,7 @@ import asyncio import logging +from enum import Enum import httpx @@ -38,12 +39,18 @@ _BASE_POINTS_PER_HOUR = 50 +class _GrassStatus(Enum): + AUTH_FAILED = "auth_failed" + RATE_LIMITED = "rate_limited" + + class GrassCollector(BaseCollector): """Collect earnings from Grass's API using an access token.""" platform = "grass" def __init__(self, access_token: str) -> None: + super().__init__() self.access_token = access_token def _make_headers(self) -> dict[str, str]: @@ -70,16 +77,16 @@ async def _request_with_retry(self, client: httpx.AsyncClient, url: str, **kwarg await asyncio.sleep(wait) return resp # Return last 429 response if all retries exhausted - async def _get_settled_points(self, client: httpx.AsyncClient) -> float: + async def _get_settled_points(self, client: httpx.AsyncClient) -> float | _GrassStatus: """Fetch totalPoints from /retrieveUser (non-zero only after epoch settles).""" resp = await self._request_with_retry(client, f"{API_BASE}/retrieveUser") if resp.status_code in (401, 403): - return -1.0 # Signal auth failure + return _GrassStatus.AUTH_FAILED if resp.status_code == 429: logger.warning("Grass API still rate-limited after retries") - return -2.0 # Signal rate limit + return _GrassStatus.RATE_LIMITED resp.raise_for_status() data = resp.json() @@ -145,49 +152,49 @@ async def collect(self) -> EarningsResult: 3. Otherwise, estimate from /devices uptime data. """ try: - async with httpx.AsyncClient(timeout=60) as client: - # First, check if we have settled points - settled = await self._get_settled_points(client) + client = self._get_client(timeout=60) - if settled == -1.0: - return EarningsResult( - platform=self.platform, - balance=0.0, - error="Token expired — get a new accessToken from app.grass.io Local Storage", - ) + # First, check if we have settled points + settled = await self._get_settled_points(client) - if settled == -2.0: + if isinstance(settled, _GrassStatus): + if settled is _GrassStatus.AUTH_FAILED: return EarningsResult( platform=self.platform, balance=0.0, - error="Cloudflare rate limit — will retry next collection cycle", - ) - - if settled > 0: - # Epoch has settled, use the official total - return EarningsResult( - platform=self.platform, - balance=round(settled, 4), - currency="GRASS", + error="Token expired — get a new accessToken from app.grass.io Local Storage", ) + return EarningsResult( + platform=self.platform, + balance=0.0, + error="Cloudflare rate limit — will retry next collection cycle", + ) - # Active epoch: estimate from active device uptime - estimated = await self._estimate_from_active_devices(client) + if settled > 0: + # Epoch has settled, use the official total + return EarningsResult( + platform=self.platform, + balance=round(settled, 4), + currency="GRASS", + ) - if estimated == -1.0: - return EarningsResult( - platform=self.platform, - balance=0.0, - error="Cloudflare rate limit — will retry next collection cycle", - ) + # Active epoch: estimate from active device uptime + estimated = await self._estimate_from_active_devices(client) + if estimated == -1.0: return EarningsResult( platform=self.platform, - balance=round(estimated, 4), - currency="GRASS", + balance=0.0, + error="Cloudflare rate limit — will retry next collection cycle", ) + + return EarningsResult( + platform=self.platform, + balance=round(estimated, 4), + currency="GRASS", + ) except Exception as exc: - logger.error("Grass collection failed: %s", exc) + logger.error("Grass collection failed: %s", exc, exc_info=True) return EarningsResult( platform=self.platform, balance=0.0, diff --git a/app/collectors/honeygain.py b/app/collectors/honeygain.py index 7e71f21..e8bfc60 100644 --- a/app/collectors/honeygain.py +++ b/app/collectors/honeygain.py @@ -21,8 +21,10 @@ class HoneygainCollector(BaseCollector): """Collect earnings from Honeygain's REST API.""" platform = "honeygain" + _HEADERS = {"User-Agent": "Mozilla/5.0", "Referer": "https://dashboard.honeygain.com/"} def __init__(self, email: str, password: str) -> None: + super().__init__() self.email = email self.password = password self._token: str | None = None @@ -32,10 +34,7 @@ async def _authenticate(self, client: httpx.AsyncClient) -> str: resp = await client.post( f"{API_BASE}/v1/users/tokens", json={"email": self.email, "password": self.password}, - headers={ - "User-Agent": "Mozilla/5.0", - "Referer": "https://dashboard.honeygain.com/", - }, + headers=self._HEADERS, ) resp.raise_for_status() data = resp.json() @@ -47,11 +46,13 @@ async def _authenticate(self, client: httpx.AsyncClient) -> str: async def collect(self) -> EarningsResult: """Fetch current Honeygain balance.""" try: - async with httpx.AsyncClient(timeout=30) as client: - if not self._token: - self._token = await self._authenticate(client) + client = self._get_client(timeout=30) + + if not self._token: + self._token = await self._authenticate(client) - headers = {"Authorization": f"Bearer {self._token}"} + async def _fetch_balance(): + headers = {**self._HEADERS, "Authorization": f"Bearer {self._token}"} resp = await client.get( f"{API_BASE}/v1/users/balances", headers=headers, @@ -60,27 +61,30 @@ async def collect(self) -> EarningsResult: # Token may have expired — retry once if resp.status_code == 401: self._token = await self._authenticate(client) - headers = {"Authorization": f"Bearer {self._token}"} + headers = {**self._HEADERS, "Authorization": f"Bearer {self._token}"} resp = await client.get( f"{API_BASE}/v1/users/balances", headers=headers, ) resp.raise_for_status() - data = resp.json() + return resp - # Balance is in cents (usd_cents) - payout = data.get("data", {}).get("payout", {}) - usd_cents = float(payout.get("usd_cents", 0)) - balance_usd = round(usd_cents / 100, 4) + resp = await self._retry(_fetch_balance) + data = resp.json() - return EarningsResult( - platform=self.platform, - balance=balance_usd, - currency="USD", - ) + # Balance is in cents (usd_cents) + payout = data.get("data", {}).get("payout", {}) + usd_cents = float(payout.get("usd_cents", 0)) + balance_usd = round(usd_cents / 100, 4) + + return EarningsResult( + platform=self.platform, + balance=balance_usd, + currency="USD", + ) except Exception as exc: - logger.error("Honeygain collection failed: %s", exc) + logger.error("Honeygain collection failed: %s", exc, exc_info=True) return EarningsResult( platform=self.platform, balance=0.0, diff --git a/app/collectors/iproyal.py b/app/collectors/iproyal.py index 09eab84..6ab403c 100644 --- a/app/collectors/iproyal.py +++ b/app/collectors/iproyal.py @@ -31,15 +31,18 @@ class IPRoyalCollector(BaseCollector): platform = "iproyal" def __init__(self, email: str, password: str) -> None: + super().__init__() self.email = email self.password = password + self._device_id = _generate_identifier() + self._token: str | None = None async def _login(self, client: httpx.AsyncClient) -> str | None: """Login and return a JWT access token, or None on failure.""" resp = await client.post( f"{API_BASE}/users/tokens", json={ - "identifier": _generate_identifier(), + "identifier": self._device_id, "email": self.email, "password": self.password, "h_captcha_response": "", @@ -57,41 +60,51 @@ async def _login(self, client: httpx.AsyncClient) -> str | None: data = resp.json() return data.get("access_token") + async def _fetch_balance(self, client: httpx.AsyncClient) -> httpx.Response: + """Fetch balance dashboard (used as retry target).""" + return await client.get( + f"{API_BASE}/users/me/balance-dashboard", + headers={"Authorization": f"Bearer {self._token}"}, + ) + async def collect(self) -> EarningsResult: """Fetch current IPRoyal Pawns balance.""" try: - async with httpx.AsyncClient(timeout=30) as client: - token = await self._login(client) - if not token: - return EarningsResult( - platform=self.platform, - balance=0.0, - error="Login failed — check email/password", - ) + client = self._get_client(timeout=30) - resp = await client.get( - f"{API_BASE}/users/me/balance-dashboard", - headers={"Authorization": f"Bearer {token}"}, + if not self._token: + self._token = await self._login(client) + if not self._token: + return EarningsResult( + platform=self.platform, + balance=0.0, + error="Login failed — check email/password", ) - if resp.status_code == 401: + resp = await self._retry(lambda: self._fetch_balance(client)) + + if resp.status_code == 401: + # Token expired, re-login once + self._token = await self._login(client) + if not self._token: return EarningsResult( platform=self.platform, balance=0.0, - error="Authentication expired", + error="Re-login failed after 401", ) + resp = await self._retry(lambda: self._fetch_balance(client)) - resp.raise_for_status() - data = resp.json() - balance = float(data.get("balance", 0)) + resp.raise_for_status() + data = resp.json() + balance = float(data.get("balance", 0)) - return EarningsResult( - platform=self.platform, - balance=round(balance, 4), - currency="USD", - ) + return EarningsResult( + platform=self.platform, + balance=round(balance, 4), + currency="USD", + ) except Exception as exc: - logger.error("IPRoyal collection failed: %s", exc) + logger.error("IPRoyal collection failed: %s", exc, exc_info=True) return EarningsResult( platform=self.platform, balance=0.0, diff --git a/app/collectors/mystnodes.py b/app/collectors/mystnodes.py index 1f9e17c..a18fc3a 100644 --- a/app/collectors/mystnodes.py +++ b/app/collectors/mystnodes.py @@ -28,6 +28,7 @@ def __init__( email: str = "", password: str = "", ) -> None: + super().__init__() self.email = email self.password = password self._access_token: str | None = None @@ -92,21 +93,21 @@ async def collect(self) -> EarningsResult: error="MystNodes email/password not configured", ) try: - async with httpx.AsyncClient(timeout=30) as client: - resp = await self._get_with_retry(client, f"{CLOUD_BASE}/node/total-earnings") - resp.raise_for_status() - data = resp.json() - - # earningsTotal is in MYST tokens (Polygon) - total_myst = float(data.get("earningsTotal", 0)) - - return EarningsResult( - platform=self.platform, - balance=round(total_myst, 4), - currency="MYST", - ) + client = self._get_client(timeout=30) + resp = await self._retry(lambda: self._get_with_retry(client, f"{CLOUD_BASE}/node/total-earnings")) + resp.raise_for_status() + data = resp.json() + + # earningsTotal is in MYST tokens (Polygon) + total_myst = float(data.get("earningsTotal", 0)) + + return EarningsResult( + platform=self.platform, + balance=round(total_myst, 4), + currency="MYST", + ) except Exception as exc: - logger.error("MystNodes collection failed: %s", exc) + logger.error("MystNodes collection failed: %s", exc, exc_info=True) return EarningsResult( platform=self.platform, balance=0.0, @@ -129,40 +130,42 @@ async def get_per_node_earnings(self) -> list[dict[str, Any]]: if not self.email or not self.password: return [] try: - async with httpx.AsyncClient(timeout=30) as client: - resp = await self._get_with_retry( + client = self._get_client(timeout=30) + resp = await self._retry( + lambda: self._get_with_retry( client, f"{CLOUD_BASE}/node", params={"page": 1, "itemsPerPage": 100}, ) - resp.raise_for_status() - data = resp.json() - - result = [] - for node in data.get("nodes", []): - # Sum 30-day earnings across all service types - earnings_30d = sum(float(e.get("etherAmount", 0)) for e in node.get("earnings", [])) - - lifetime = node.get("lifetimeEarnings") or {} - total_ether = float(lifetime.get("totalEther", 0)) - settled = float(lifetime.get("settledEther", 0)) - unsettled = float(lifetime.get("unsettledEther", 0)) - - result.append( - { - "identity": node.get("identity", ""), - "name": node.get("name") or "", - "local_ip": node.get("localIp", ""), - "online": (node.get("nodeStatus", {}).get("online", False)), - "country": (node.get("country", {}).get("code", "")), - "version": node.get("version", ""), - "earnings_myst": round(earnings_30d, 6), - "lifetime_myst": round(total_ether, 6), - "lifetime_settled_myst": round(settled, 6), - "lifetime_unsettled_myst": round(unsettled, 6), - } - ) - return result + ) + resp.raise_for_status() + data = resp.json() + + result = [] + for node in data.get("nodes", []): + # Sum 30-day earnings across all service types + earnings_30d = sum(float(e.get("etherAmount", 0)) for e in node.get("earnings", [])) + + lifetime = node.get("lifetimeEarnings") or {} + total_ether = float(lifetime.get("totalEther", 0)) + settled = float(lifetime.get("settledEther", 0)) + unsettled = float(lifetime.get("unsettledEther", 0)) + + result.append( + { + "identity": node.get("identity", ""), + "name": node.get("name") or "", + "local_ip": node.get("localIp", ""), + "online": (node.get("nodeStatus", {}).get("online", False)), + "country": (node.get("country", {}).get("code", "")), + "version": node.get("version", ""), + "earnings_myst": round(earnings_30d, 6), + "lifetime_myst": round(total_ether, 6), + "lifetime_settled_myst": round(settled, 6), + "lifetime_unsettled_myst": round(unsettled, 6), + } + ) + return result except Exception as exc: - logger.error("MystNodes per-node fetch failed: %s", exc) + logger.error("MystNodes per-node fetch failed: %s", exc, exc_info=True) return [] diff --git a/app/collectors/packetstream.py b/app/collectors/packetstream.py index 408fc94..e0d9856 100644 --- a/app/collectors/packetstream.py +++ b/app/collectors/packetstream.py @@ -6,6 +6,7 @@ from __future__ import annotations +import json import logging import re @@ -24,77 +25,79 @@ class PacketStreamCollector(BaseCollector): platform = "packetstream" def __init__(self, auth_token: str) -> None: + super().__init__() self.auth_token = auth_token async def collect(self) -> EarningsResult: """Fetch current PacketStream balance by scraping dashboard.""" try: cookies = {"auth": self.auth_token} + client = self._get_client(cookies=cookies) - async with httpx.AsyncClient(timeout=30, cookies=cookies) as client: - resp = await client.get(f"{API_BASE}/dashboard") + async def _fetch() -> httpx.Response: + return await client.get(f"{API_BASE}/dashboard") - if resp.status_code in (401, 403) or "/login" in str(resp.url): - return EarningsResult( - platform=self.platform, - balance=0.0, - error="Authentication failed — check auth JWT cookie", - ) + resp = await self._retry(_fetch) - resp.raise_for_status() - html = resp.text + if resp.status_code in (401, 403) or "/login" in str(resp.url): + return EarningsResult( + platform=self.platform, + balance=0.0, + error="Authentication failed — check auth JWT cookie", + ) + + resp.raise_for_status() + html = resp.text - balance = 0.0 - parsed = False + balance = 0.0 + parsed = False - # Pattern 1: server-rendered Balance card (current) - #
no balance
", url="https://dash.bytelixir.com/en") + html_resp.url = url_mock html_resp.text = "no balance
" # API fallback succeeds api_resp = _mock_response(200, {"data": {"balance": "0.5000000000"}}) @@ -420,7 +411,7 @@ def test_collect_api_fallback(self): client = _make_async_client() client.get.side_effect = [html_resp, api_resp] - with patch.object(BytelixirCollector, "_make_client", return_value=client): + with patch.object(BytelixirCollector, "_get_client", return_value=client): c = BytelixirCollector(session_cookie="valid") result = asyncio.run(c.collect()) assert result.balance == 0.50 @@ -428,7 +419,10 @@ def test_collect_api_fallback(self): def test_collect_api_401(self): from app.collectors.bytelixir import BytelixirCollector + url_mock = MagicMock() + url_mock.path = "/en" html_resp = _mock_response(200, text="no balance
", url="https://dash.bytelixir.com/en") + html_resp.url = url_mock html_resp.text = "no balance
" api_resp = _mock_response(401) api_resp.raise_for_status = MagicMock() # 401 handled inline @@ -436,7 +430,7 @@ def test_collect_api_401(self): client = _make_async_client() client.get.side_effect = [html_resp, api_resp] - with patch.object(BytelixirCollector, "_make_client", return_value=client): + with patch.object(BytelixirCollector, "_get_client", return_value=client): c = BytelixirCollector(session_cookie="expired") result = asyncio.run(c.collect()) assert result.error is not None diff --git a/tests/test_collectors_extended.py b/tests/test_collectors_extended.py index fedf804..2c05b1f 100644 --- a/tests/test_collectors_extended.py +++ b/tests/test_collectors_extended.py @@ -494,11 +494,14 @@ class TestBytelixirCollector: def test_collect_session_expired(self): from app.collectors.bytelixir import BytelixirCollector + url_mock = MagicMock() + url_mock.path = "/login" resp = _mock_response(200, url="https://dash.bytelixir.com/login") + resp.url = url_mock client = _make_async_client() client.get.return_value = resp - with patch.object(BytelixirCollector, "_make_client", return_value=client): + with patch.object(BytelixirCollector, "_get_client", return_value=client): c = BytelixirCollector(session_cookie="expired") result = asyncio.run(c.collect()) assert result.error is not None @@ -508,12 +511,15 @@ def test_collect_html_scrape_success(self): from app.collectors.bytelixir import BytelixirCollector html = '$0.04025' + url_mock = MagicMock() + url_mock.path = "/en" resp = _mock_response(200, text=html, url="https://dash.bytelixir.com/en") + resp.url = url_mock resp.text = html client = _make_async_client() client.get.return_value = resp - with patch.object(BytelixirCollector, "_make_client", return_value=client): + with patch.object(BytelixirCollector, "_get_client", return_value=client): c = BytelixirCollector(session_cookie="valid-sess") result = asyncio.run(c.collect()) assert result.error is None @@ -525,7 +531,7 @@ def test_collect_error(self): client = _make_async_client() client.get.side_effect = Exception("Error") - with patch.object(BytelixirCollector, "_make_client", return_value=client): + with patch.object(BytelixirCollector, "_get_client", return_value=client): c = BytelixirCollector(session_cookie="bad") result = asyncio.run(c.collect()) assert result.error is not None diff --git a/tests/test_coverage_gaps.py b/tests/test_coverage_gaps.py index d64477d..1d0f753 100644 --- a/tests/test_coverage_gaps.py +++ b/tests/test_coverage_gaps.py @@ -10,10 +10,11 @@ from contextlib import asynccontextmanager from unittest.mock import AsyncMock, MagicMock, patch +import pytest + os.environ.setdefault("CASHPILOT_API_KEY", "test-fleet-key") import httpx -import pytest import yaml from app import catalog, database @@ -217,14 +218,14 @@ def test_deployment_without_slug_key(self): # --------------------------------------------------------------------------- -# bytelixir.py — _make_client, parse balance edge cases +# bytelixir.py — _get_client, parse balance edge cases # --------------------------------------------------------------------------- -class TestBytelixirMakeClient: - """Cover lines 59-77: _make_client with remember_web and xsrf_token.""" +class TestBytelixirGetClient: + """Cover _get_client with remember_web and xsrf_token.""" - def test_make_client_with_all_cookies(self): + def test_get_client_with_all_cookies(self): from app.collectors.bytelixir import BytelixirCollector c = BytelixirCollector( @@ -232,7 +233,7 @@ def test_make_client_with_all_cookies(self): remember_web="remember-val", xsrf_token="xsrf-val", ) - client = c._make_client() + client = c._get_client() assert isinstance(client, httpx.AsyncClient) # Verify cookies are set cookies = dict(client.cookies) @@ -241,11 +242,11 @@ def test_make_client_with_all_cookies(self): assert "XSRF-TOKEN" in cookies asyncio.run(client.aclose()) - def test_make_client_session_only(self): + def test_get_client_session_only(self): from app.collectors.bytelixir import BytelixirCollector c = BytelixirCollector(session_cookie="sess-only") - client = c._make_client() + client = c._get_client() cookies = dict(client.cookies) assert "bytelixir_session" in cookies assert c._REMEMBER_COOKIE not in cookies @@ -914,18 +915,20 @@ def test_bitping_login_missing_cookie(self): result = asyncio.run(c.collect()) assert result.error is not None - def test_traffmonetizer_login_missing_token(self): - """Cover traffmonetizer.py line 45: login response without token.""" + def test_traffmonetizer_403_returns_expired_error(self): + """Cover traffmonetizer.py: 403 response returns token expired error.""" from app.collectors.traffmonetizer import TraffmonetizerCollector - login_resp = _mock_response(200, {"data": {}}) + resp_403 = MagicMock() + resp_403.status_code = 403 client = _make_async_client() - client.post.return_value = login_resp + client.get.return_value = resp_403 with patch("app.collectors.traffmonetizer.httpx.AsyncClient", return_value=client): - c = TraffmonetizerCollector(email="test@test.com", password="pass") + c = TraffmonetizerCollector(token="some-jwt") result = asyncio.run(c.collect()) assert isinstance(result, EarningsResult) + assert "expired" in result.error.lower() def test_repocket_login_missing_token(self): """Cover repocket.py lines 49, 55: login without idToken.""" @@ -1087,3 +1090,116 @@ def test_resolve_secret_key_oserror_reading(self, tmp_path): result = auth._resolve_secret_key() # Should generate a new key since reading failed assert len(result) > 20 + + +# --------------------------------------------------------------------------- +# Orchestrator: volume cleanup on remove +# --------------------------------------------------------------------------- + + +docker = pytest.importorskip("docker") + + +class TestOrchestratorVolumeCleanup: + """Cover orchestrator.remove_service with delete_volumes flag.""" + + def test_remove_without_volumes(self): + from app import orchestrator + + container = MagicMock() + container.name = "cashpilot-test" + container.attrs = {"Mounts": []} + + with patch.object(orchestrator, "_find_container", return_value=container): + result = orchestrator.remove_service("test") + + container.remove.assert_called_once_with(force=True) + assert result["container"] == "cashpilot-test" + assert result["deleted_volumes"] == [] + assert result["failed_volumes"] == [] + + def test_remove_with_delete_volumes_named(self): + from app import orchestrator + + container = MagicMock() + container.name = "cashpilot-honeygain" + container.attrs = { + "Mounts": [ + {"Type": "volume", "Name": "honeygain_data"}, + {"Type": "bind", "Source": "/host/path"}, + {"Type": "volume", "Name": "honeygain_config"}, + ] + } + + mock_vol = MagicMock() + mock_client = MagicMock() + mock_client.volumes.get.return_value = mock_vol + + with ( + patch.object(orchestrator, "_find_container", return_value=container), + patch.object(orchestrator, "_get_client", return_value=mock_client), + ): + result = orchestrator.remove_service("honeygain", delete_volumes=True) + + container.remove.assert_called_once_with(force=True) + assert "honeygain_data" in result["deleted_volumes"] + assert "honeygain_config" in result["deleted_volumes"] + assert result["failed_volumes"] == [] + assert mock_vol.remove.call_count == 2 + + def test_remove_volume_not_found_counts_as_deleted(self): + from docker.errors import NotFound + + from app import orchestrator + + container = MagicMock() + container.name = "cashpilot-test" + container.attrs = {"Mounts": [{"Type": "volume", "Name": "gone_vol"}]} + + mock_client = MagicMock() + mock_client.volumes.get.side_effect = NotFound("gone") + + with ( + patch.object(orchestrator, "_find_container", return_value=container), + patch.object(orchestrator, "_get_client", return_value=mock_client), + ): + result = orchestrator.remove_service("test", delete_volumes=True) + + assert "gone_vol" in result["deleted_volumes"] + assert result["failed_volumes"] == [] + + def test_remove_volume_api_error_recorded(self): + from docker.errors import APIError + + from app import orchestrator + + container = MagicMock() + container.name = "cashpilot-test" + container.attrs = {"Mounts": [{"Type": "volume", "Name": "busy_vol"}]} + + mock_vol = MagicMock() + mock_vol.remove.side_effect = APIError("volume in use") + mock_client = MagicMock() + mock_client.volumes.get.return_value = mock_vol + + with ( + patch.object(orchestrator, "_find_container", return_value=container), + patch.object(orchestrator, "_get_client", return_value=mock_client), + ): + result = orchestrator.remove_service("test", delete_volumes=True) + + assert result["deleted_volumes"] == [] + assert "busy_vol" in result["failed_volumes"] + + def test_delete_volumes_false_skips_volume_inspection(self): + from app import orchestrator + + container = MagicMock() + container.name = "cashpilot-test" + container.attrs = {"Mounts": [{"Type": "volume", "Name": "keep_me"}]} + + with patch.object(orchestrator, "_find_container", return_value=container): + result = orchestrator.remove_service("test", delete_volumes=False) + + assert result["deleted_volumes"] == [] + assert result["failed_volumes"] == [] diff --git a/tests/test_main_deploy_routes.py b/tests/test_main_deploy_routes.py index ede5a44..d258130 100644 --- a/tests/test_main_deploy_routes.py +++ b/tests/test_main_deploy_routes.py @@ -269,6 +269,20 @@ def test_old_remove_route(self, client): resp = client.delete("/api/remove/honeygain") assert resp.status_code == 200 + def test_remove_with_delete_volumes(self, client): + worker, mock_client = self._setup_proxy() + with ( + _auth_writer(), + patch("app.main.database.list_workers", new_callable=AsyncMock, return_value=[worker]), + patch("app.main.database.get_worker", new_callable=AsyncMock, return_value=worker), + patch("app.main.database.remove_deployment", new_callable=AsyncMock), + patch("app.main.httpx.AsyncClient", return_value=mock_client), + patch("app.main.FLEET_API_KEY", "test-key"), + ): + resp = client.delete("/api/services/honeygain?delete_volumes=true") + assert resp.status_code == 200 + assert mock_client.delete.call_args.kwargs["params"] == {"delete_volumes": "true"} + # --------------------------------------------------------------------------- # Worker proxy error handling