feat(monitoring): add Prometheus metrics endpoint and instrumentation

rondinelisaad · rondinelisaad · commit cad9107af654 · 2026-05-22T16:32:59.000-03:00
diff --git a/analytics/__init__.py b/analytics/__init__.py
@@ -2,12 +2,14 @@
 import time
 import logging
 import threading
+from pyramid.response import Response
 from pyramid.session import SignedCookieSessionFactory
 from pyramid.renderers import JSONP
 from pyramid.config import Configurator
 
 from analytics import controller
 from analytics import charts_config
+from analytics import monitoring
 
 from analytics.views_website import cache_region as views_website_cache_region
 from analytics.views_ajax import cache_region as views_ajax_cache_region
@@ -17,7 +19,39 @@
 logger = logging.getLogger(__name__)
 
 
-def _start_cache_prewarm(settings):
+def _metrics_view(_request):
+    body, content_type = monitoring.metrics_response()
+    return Response(body=body, content_type=content_type)
+
+
+def _request_metrics_tween_factory(handler, registry):
+    def _tween(request):
+        route_name = "unknown"
+        if request.matched_route is not None:
+            route_name = request.matched_route.name
+        method = request.method
+        timer = monitoring.HTTP_REQUEST_DURATION_SECONDS.labels(method=method, route=route_name).time()
+        monitoring.HTTP_REQUESTS_IN_PROGRESS.labels(method=method, route=route_name).inc()
+        try:
+            with timer:
+                response = handler(request)
+            status_code = str(getattr(response, "status_code", "200"))
+            monitoring.HTTP_REQUESTS_TOTAL.labels(
+                method=method, route=route_name, status_code=status_code
+            ).inc()
+            return response
+        except Exception:
+            monitoring.HTTP_REQUESTS_TOTAL.labels(
+                method=method, route=route_name, status_code="500"
+            ).inc()
+            raise
+        finally:
+            monitoring.HTTP_REQUESTS_IN_PROGRESS.labels(method=method, route=route_name).dec()
+
+    return _tween
+
+
+def _start_cache_prewarm(settings, usage_api_base_url=None):
     if os.environ.get("ENABLE_CACHE_PREWARM", "1").lower() in ("0", "false", "no"):
         return
 
@@ -32,7 +66,7 @@ def _prewarm():
                 settings.get('articlemeta', None),
                 settings.get('publicationstats', None),
                 settings.get('citedby', None),
-                settings.get('usage', None),
+                usage_api_base_url,
                 request=None
             )
             stats.articlemeta.certified_collections()
@@ -71,13 +105,14 @@ def main(global_config, **settings):
     """
     config = Configurator(settings=settings)
     config.add_renderer('jsonp', JSONP(param_name='callback', indent=4))
+    usage_api_base_url = os.environ.get('USAGE_API_BASE_URL', settings.get('usage', None))
 
     def add_stats(request):
         return controller.Stats(
             settings.get('articlemeta', None),
             settings.get('publicationstats', None),
             settings.get('citedby', None),
-            settings.get('usage', None),
+            usage_api_base_url,
             request
         )
 
@@ -89,6 +124,7 @@ def add_chartsconfig(request):
     config.add_route('index_web', '/')
     config.add_route('faq_web', '/w/faq')
     config.add_route('reports', '/w/reports')
+    config.add_route('metrics', '/metrics')
     config.add_route('usage_report_chart', '/ajx/usage/usage_report_chart')
     config.add_route('usage_report_yearly_chart', '/ajx/usage/usage_report_yearly_chart')
     config.add_route('accesses_web', '/w/accesses')
@@ -138,6 +174,8 @@ def add_chartsconfig(request):
     config.add_subscriber('analytics.subscribers.add_localizer',
                           'pyramid.events.NewRequest')
     config.add_translation_dirs('analytics:locale')
+    config.add_tween('analytics._request_metrics_tween_factory', over='MAIN')
+    config.add_view(_metrics_view, route_name='metrics', request_method='GET')
 
     # Cache Settings Config
     memcached_host = (
@@ -162,6 +200,8 @@ def add_chartsconfig(request):
         views_website_cache_region.configure('dogpile.cache.pylibmc', **cache_config)
         controller_cache_region.configure('dogpile.cache.pylibmc', **cache_config)
         control_manager_cache_region.configure('dogpile.cache.pylibmc', **cache_config)
+        monitoring.CACHE_BACKEND_INFO.labels(backend='pylibmc').set(1)
+        monitoring.CACHE_BACKEND_INFO.labels(backend='memory').set(0)
     else:
         # Fallback to in-process cache when memcached is unavailable.
         # This avoids recomputing heavy remote calls on every request.
@@ -170,11 +210,13 @@ def add_chartsconfig(request):
         controller_cache_region.configure('dogpile.cache.memory', **cache_config)
         control_manager_cache_region.configure('dogpile.cache.memory', **cache_config)
         views_ajax_cache_region.configure('dogpile.cache.memory', **cache_config)
+        monitoring.CACHE_BACKEND_INFO.labels(backend='memory').set(1)
+        monitoring.CACHE_BACKEND_INFO.labels(backend='pylibmc').set(0)
 
     # Session config
     navegation_session_factory = SignedCookieSessionFactory('sses_navegation')
     config.set_session_factory(navegation_session_factory)
-    _start_cache_prewarm(settings)
+    _start_cache_prewarm(settings, usage_api_base_url=usage_api_base_url)
 
     config.scan()
 
diff --git a/analytics/monitoring.py b/analytics/monitoring.py
@@ -0,0 +1,77 @@
+import time
+
+from prometheus_client import (
+    CONTENT_TYPE_LATEST,
+    Counter,
+    Gauge,
+    Histogram,
+    generate_latest,
+)
+
+
+HTTP_REQUESTS_TOTAL = Counter(
+    "analytics_http_requests_total",
+    "Total HTTP requests handled by the app",
+    ["method", "route", "status_code"],
+)
+
+HTTP_REQUEST_DURATION_SECONDS = Histogram(
+    "analytics_http_request_duration_seconds",
+    "HTTP request latency in seconds",
+    ["method", "route"],
+)
+
+HTTP_REQUESTS_IN_PROGRESS = Gauge(
+    "analytics_http_requests_in_progress",
+    "HTTP requests currently in progress",
+    ["method", "route"],
+)
+
+BACKEND_CALLS_TOTAL = Counter(
+    "analytics_backend_calls_total",
+    "Total backend calls by backend and operation",
+    ["backend", "operation", "result"],
+)
+
+BACKEND_CALL_DURATION_SECONDS = Histogram(
+    "analytics_backend_call_duration_seconds",
+    "Backend call duration in seconds",
+    ["backend", "operation"],
+)
+
+BACKEND_INFLIGHT_REJECTED_TOTAL = Counter(
+    "analytics_backend_inflight_rejected_total",
+    "Rejected backend calls due to inflight guard saturation",
+    ["backend", "operation"],
+)
+
+CIRCUIT_BREAKER_STATE = Gauge(
+    "analytics_circuit_breaker_state",
+    "Circuit breaker state (1=open, 0=closed)",
+    ["host"],
+)
+
+CACHE_BACKEND_INFO = Gauge(
+    "analytics_cache_backend_info",
+    "Configured cache backend (1 for active backend)",
+    ["backend"],
+)
+
+
+def metrics_response():
+    return generate_latest(), CONTENT_TYPE_LATEST
+
+
+def observe_backend_call(backend, operation, callable_obj):
+    started_at = time.monotonic()
+    try:
+        result = callable_obj()
+        BACKEND_CALLS_TOTAL.labels(backend=backend, operation=operation, result="success").inc()
+        return result
+    except Exception:
+        BACKEND_CALLS_TOTAL.labels(backend=backend, operation=operation, result="error").inc()
+        raise
+    finally:
+        BACKEND_CALL_DURATION_SECONDS.labels(backend=backend, operation=operation).observe(
+            time.monotonic() - started_at
+        )
diff --git a/analytics/request_utils.py b/analytics/request_utils.py
@@ -5,6 +5,7 @@
 from urllib.parse import urlparse
 
 import requests
+from analytics import monitoring
 
 from requests.exceptions import (
     ConnectionError,
@@ -68,6 +69,7 @@ def _breaker_mark_success():
     with _breaker_lock:
         _breaker_state["failures"] = 0
         _breaker_state["open_until"] = 0.0
+        monitoring.CIRCUIT_BREAKER_STATE.labels(host=CIRCUIT_BREAKER_HOST).set(0)
 
 
 def _breaker_mark_failure():
@@ -76,6 +78,7 @@ def _breaker_mark_failure():
         failures = _breaker_state["failures"]
         if failures >= CIRCUIT_BREAKER_FAILURE_THRESHOLD:
             _breaker_state["open_until"] = time.time() + CIRCUIT_BREAKER_OPEN_SECONDS
+            monitoring.CIRCUIT_BREAKER_STATE.labels(host=CIRCUIT_BREAKER_HOST).set(1)
             logger.warning(
                 "Circuit breaker opened for host '%s' for %ss after %s failures",
                 CIRCUIT_BREAKER_HOST,
@@ -123,40 +126,46 @@ def fetch_data(url, json=True, params=None, timeout=32, verify=True):
         RetryableError: If a connection or timeout error occurs (for retry).
         NonRetryableError: For schema, URL, or 4xx client errors.
     """
-    if _is_breaker_target(url) and _breaker_is_open():
-        logger.warning(
-            "Circuit breaker open for host '%s', short-circuiting request: %s",
-            CIRCUIT_BREAKER_HOST,
-            url,
-        )
-        raise NonRetryableError(CircuitOpenError(url))
+    def _do_fetch():
+        if _is_breaker_target(url) and _breaker_is_open():
+            logger.warning(
+                "Circuit breaker open for host '%s', short-circuiting request: %s",
+                CIRCUIT_BREAKER_HOST,
+                url,
+            )
+            monitoring.BACKEND_CALLS_TOTAL.labels(
+                backend="usage_api", operation="fetch_data", result="circuit_open"
+            ).inc()
+            raise NonRetryableError(CircuitOpenError(url))
+
+        try:
+            logger.info(f"Fetching URL: {url} with params {params}")
+            response = requests.get(url, params=params, timeout=timeout, verify=verify)
+            response.raise_for_status()
+            if _is_breaker_target(url):
+                _breaker_mark_success()
 
-    try:
-        logger.info(f"Fetching URL: {url} with params {params}")
-        response = requests.get(url, params=params, timeout=timeout, verify=verify)
-        response.raise_for_status()
-        if _is_breaker_target(url):
-            _breaker_mark_success()
-
-    except (ConnectionError, Timeout) as exc:
-        if _is_breaker_target(url):
-            _breaker_mark_failure()
-        logger.error(f"Erro fetching content: {url}. Retrying... Error: {exc}")
-        raise RetryableError(exc) from exc
-
-    except (InvalidSchema, MissingSchema, InvalidURL) as exc:
-        logger.error(f"Invalid URL or schema: {url}. Error: {exc}")
-        raise NonRetryableError(exc) from exc
-    
-    except HTTPError as exc:
-        status_code = exc.response.status_code
-        if 400 <= status_code < 500:
-            logger.error(f"Client error (non-retryable): {url}. Status: {status_code}")
-            raise NonRetryableError(exc) from exc
-        elif 500 <= status_code < 600:
+        except (ConnectionError, Timeout) as exc:
             if _is_breaker_target(url):
                 _breaker_mark_failure()
-            logger.error(f"Server error: {url}. Retrying... Status: {status_code}")
+            logger.error(f"Erro fetching content: {url}. Retrying... Error: {exc}")
             raise RetryableError(exc) from exc
 
-    return response.json() if json else response.content
+        except (InvalidSchema, MissingSchema, InvalidURL) as exc:
+            logger.error(f"Invalid URL or schema: {url}. Error: {exc}")
+            raise NonRetryableError(exc) from exc
+        
+        except HTTPError as exc:
+            status_code = exc.response.status_code
+            if 400 <= status_code < 500:
+                logger.error(f"Client error (non-retryable): {url}. Status: {status_code}")
+                raise NonRetryableError(exc) from exc
+            elif 500 <= status_code < 600:
+                if _is_breaker_target(url):
+                    _breaker_mark_failure()
+                logger.error(f"Server error: {url}. Retrying... Status: {status_code}")
+                raise RetryableError(exc) from exc
+
+        return response.json() if json else response.content
+
+    return monitoring.observe_backend_call("usage_api", "fetch_data", _do_fetch)
diff --git a/analytics/views_ajax.py b/analytics/views_ajax.py
