Introduce telemetry for observability

This introduces the foundational telemetry infrastructure to improve
the observability of LDK Server.

It adds a new `/metrics` endpoint exposed on the REST service address,
which serves Prometheus-compatible metrics. This endpoint is public and
does not require HMAC authentication, allowing for easy integration with
monitoring systems.

- Added a `Metrics` utility struct to hold all the metrics we need to
expose.

This is the first step in a larger effort to provide comprehensive telemetry.
Future updates will expand this to include other detailed metrics for channels,
balances, payments, etc.

Author: Anyitechs

ldk-server-client/src/client.rs

@@ -29,12 +29,13 @@ use ldk_server_protos::api::{
 use ldk_server_protos::endpoints::{
 	BOLT11_RECEIVE_PATH, BOLT11_SEND_PATH, BOLT12_RECEIVE_PATH, BOLT12_SEND_PATH,
 	CLOSE_CHANNEL_PATH, CONNECT_PEER_PATH, DISCONNECT_PEER_PATH, EXPORT_PATHFINDING_SCORES_PATH,
-	FORCE_CLOSE_CHANNEL_PATH, GET_BALANCES_PATH, GET_NODE_INFO_PATH, GET_PAYMENT_DETAILS_PATH,
-	LIST_CHANNELS_PATH, LIST_FORWARDED_PAYMENTS_PATH, LIST_PAYMENTS_PATH, ONCHAIN_RECEIVE_PATH,
-	ONCHAIN_SEND_PATH, OPEN_CHANNEL_PATH, SIGN_MESSAGE_PATH, SPLICE_IN_PATH, SPLICE_OUT_PATH,
-	SPONTANEOUS_SEND_PATH, UPDATE_CHANNEL_CONFIG_PATH, VERIFY_SIGNATURE_PATH,
+	FORCE_CLOSE_CHANNEL_PATH, GET_BALANCES_PATH, GET_METRICS_PATH, GET_NODE_INFO_PATH,
+	GET_PAYMENT_DETAILS_PATH, LIST_CHANNELS_PATH, LIST_FORWARDED_PAYMENTS_PATH, LIST_PAYMENTS_PATH,
+	ONCHAIN_RECEIVE_PATH, ONCHAIN_SEND_PATH, OPEN_CHANNEL_PATH, SIGN_MESSAGE_PATH, SPLICE_IN_PATH,
+	SPLICE_OUT_PATH, SPONTANEOUS_SEND_PATH, UPDATE_CHANNEL_CONFIG_PATH, VERIFY_SIGNATURE_PATH,
 };
 use ldk_server_protos::error::{ErrorCode, ErrorResponse};
+use prost::bytes::Bytes;
 use prost::Message;
 use reqwest::header::CONTENT_TYPE;
 use reqwest::{Certificate, Client};
@@ -58,6 +59,11 @@ pub struct LdkServerClient {
 	api_key: String,
 }
 
+enum RequestType {
+	Get,
+	Post,
+}
+
 impl LdkServerClient {
 	/// Constructs a [`LdkServerClient`] using `base_url` as the ldk-server endpoint.
 	///
@@ -103,6 +109,18 @@ impl LdkServerClient {
 		self.post_request(&request, &url).await
 	}
 
+	/// Retrieve the node metrics in Prometheus format.
+	pub async fn get_metrics(&self) -> Result<String, LdkServerError> {
+		let url = format!("https://{}/{GET_METRICS_PATH}", self.base_url);
+		let payload = self.make_request(&url, RequestType::Get, None, false).await?;
+		String::from_utf8(payload.to_vec()).map_err(|e| {
+			LdkServerError::new(
+				InternalError,
+				format!("Failed to decode metrics response as string: {}", e),
+			)
+		})
+	}
+
 	/// Retrieves an overview of all known balances.
 	/// For API contract/usage, refer to docs for [`GetBalancesRequest`] and [`GetBalancesResponse`].
 	pub async fn get_balances(
@@ -314,31 +332,46 @@ impl LdkServerClient {
 		&self, request: &Rq, url: &str,
 	) -> Result<Rs, LdkServerError> {
 		let request_body = request.encode_to_vec();
-		let auth_header = self.compute_auth_header(&request_body);
-		let response_raw = self
-			.client
-			.post(url)
-			.header(CONTENT_TYPE, APPLICATION_OCTET_STREAM)
-			.header("X-Auth", auth_header)
-			.body(request_body)
-			.send()
-			.await
-			.map_err(|e| {
-				LdkServerError::new(InternalError, format!("HTTP request failed: {}", e))
-			})?;
+		let payload = self.make_request(url, RequestType::Post, Some(request_body), true).await?;
+		Rs::decode(&payload[..]).map_err(|e| {
+			LdkServerError::new(InternalError, format!("Failed to decode success response: {}", e))
+		})
+	}
+
+	async fn make_request(
+		&self, url: &str, request_type: RequestType, body: Option<Vec<u8>>, authenticated: bool,
+	) -> Result<Bytes, LdkServerError> {
+		let builder = match request_type {
+			RequestType::Get => self.client.get(url),
+			RequestType::Post => self.client.post(url),
+		};
+
+		let body_for_auth = body.as_deref().unwrap_or(&[]);
+
+		let builder = if authenticated {
+			let auth_header = self.compute_auth_header(body_for_auth);
+			builder.header("X-Auth", auth_header)
+		} else {
+			builder
+		};
+
+		let builder = if let Some(body_content) = body {
+			builder.header(CONTENT_TYPE, APPLICATION_OCTET_STREAM).body(body_content)
+		} else {
+			builder
+		};
+
+		let response_raw = builder.send().await.map_err(|e| {
+			LdkServerError::new(InternalError, format!("HTTP request failed: {}", e))
+		})?;
 
 		let status = response_raw.status();
 		let payload = response_raw.bytes().await.map_err(|e| {
 			LdkServerError::new(InternalError, format!("Failed to read response body: {}", e))
 		})?;
 
 		if status.is_success() {
-			Ok(Rs::decode(&payload[..]).map_err(|e| {
-				LdkServerError::new(
-					InternalError,
-					format!("Failed to decode success response: {}", e),
-				)
-			})?)
+			Ok(payload)
 		} else {
 			let error_response = ErrorResponse::decode(&payload[..]).map_err(|e| {
 				LdkServerError::new(

ldk-server-protos/src/endpoints.rs

@@ -31,3 +31,4 @@ pub const SPONTANEOUS_SEND_PATH: &str = "SpontaneousSend";
 pub const SIGN_MESSAGE_PATH: &str = "SignMessage";
 pub const VERIFY_SIGNATURE_PATH: &str = "VerifySignature";
 pub const EXPORT_PATHFINDING_SCORES_PATH: &str = "ExportPathfindingScores";
+pub const GET_METRICS_PATH: &str = "metrics";

ldk-server/src/main.rs

@@ -50,6 +50,7 @@ use crate::io::persist::{
 use crate::service::NodeService;
 use crate::util::config::{load_config, ArgsConfig, ChainSource};
 use crate::util::logger::ServerLogger;
+use crate::util::metrics::{Metrics, BUILD_METRICS_INTERVAL};
 use crate::util::proto_adapter::{forwarded_payment_to_proto, payment_to_proto};
 use crate::util::tls::get_or_generate_tls_config;
 
@@ -256,6 +257,20 @@ fn main() {
 			}
 		};
 		let event_node = Arc::clone(&node);
+
+		let metrics_node = Arc::clone(&node);
+		let mut interval = tokio::time::interval(BUILD_METRICS_INTERVAL);
+		let metrics = Arc::new(Metrics::new());
+		let metrics_bg = Arc::clone(&metrics);
+		let event_metrics = Arc::clone(&metrics);
+
+		runtime.spawn(async move {
+			loop {
+				interval.tick().await;
+				metrics_bg.update_all_pollable_metrics(&metrics_node);
+			}
+		});
+
 		let rest_svc_listener = TcpListener::bind(config_file.rest_service_addr)
 			.await
 			.expect("Failed to bind listening port");
@@ -320,6 +335,8 @@ fn main() {
 								&event_node,
 								Arc::clone(&event_publisher),
 								Arc::clone(&paginated_store)).await;
+
+							event_metrics.update_total_successful_payments_count(&event_node);
 						},
 						Event::PaymentFailed {payment_id, ..} => {
 							let payment_id = payment_id.expect("PaymentId expected for ldk-server >=0.1");
@@ -331,6 +348,8 @@ fn main() {
 								&event_node,
 								Arc::clone(&event_publisher),
 								Arc::clone(&paginated_store)).await;
+
+							event_metrics.update_total_failed_payments_count(&event_node);
 						},
 						Event::PaymentClaimable {payment_id, ..} => {
 							if let Some(payment_details) = event_node.payment(&payment_id) {
@@ -415,7 +434,7 @@ fn main() {
 				res = rest_svc_listener.accept() => {
 					match res {
 						Ok((stream, _)) => {
-							let node_service = NodeService::new(Arc::clone(&node), Arc::clone(&paginated_store), api_key.clone());
+							let node_service = NodeService::new(Arc::clone(&node), Arc::clone(&paginated_store), api_key.clone(), Arc::clone(&metrics));
 							let acceptor = tls_acceptor.clone();
 							runtime.spawn(async move {
 								match acceptor.accept(stream).await {

ldk-server/src/service.rs

@@ -21,10 +21,10 @@ use ldk_node::Node;
 use ldk_server_protos::endpoints::{
 	BOLT11_RECEIVE_PATH, BOLT11_SEND_PATH, BOLT12_RECEIVE_PATH, BOLT12_SEND_PATH,
 	CLOSE_CHANNEL_PATH, CONNECT_PEER_PATH, DISCONNECT_PEER_PATH, EXPORT_PATHFINDING_SCORES_PATH,
-	FORCE_CLOSE_CHANNEL_PATH, GET_BALANCES_PATH, GET_NODE_INFO_PATH, GET_PAYMENT_DETAILS_PATH,
-	LIST_CHANNELS_PATH, LIST_FORWARDED_PAYMENTS_PATH, LIST_PAYMENTS_PATH, ONCHAIN_RECEIVE_PATH,
-	ONCHAIN_SEND_PATH, OPEN_CHANNEL_PATH, SIGN_MESSAGE_PATH, SPLICE_IN_PATH, SPLICE_OUT_PATH,
-	SPONTANEOUS_SEND_PATH, UPDATE_CHANNEL_CONFIG_PATH, VERIFY_SIGNATURE_PATH,
+	FORCE_CLOSE_CHANNEL_PATH, GET_BALANCES_PATH, GET_METRICS_PATH, GET_NODE_INFO_PATH,
+	GET_PAYMENT_DETAILS_PATH, LIST_CHANNELS_PATH, LIST_FORWARDED_PAYMENTS_PATH, LIST_PAYMENTS_PATH,
+	ONCHAIN_RECEIVE_PATH, ONCHAIN_SEND_PATH, OPEN_CHANNEL_PATH, SIGN_MESSAGE_PATH, SPLICE_IN_PATH,
+	SPLICE_OUT_PATH, SPONTANEOUS_SEND_PATH, UPDATE_CHANNEL_CONFIG_PATH, VERIFY_SIGNATURE_PATH,
 };
 use prost::Message;
 
@@ -53,6 +53,7 @@ use crate::api::spontaneous_send::handle_spontaneous_send_request;
 use crate::api::update_channel_config::handle_update_channel_config_request;
 use crate::api::verify_signature::handle_verify_signature_request;
 use crate::io::persist::paginated_kv_store::PaginatedKVStore;
+use crate::util::metrics::Metrics;
 use crate::util::proto_adapter::to_error_response;
 
 // Maximum request body size: 10 MB
@@ -64,13 +65,15 @@ pub struct NodeService {
 	node: Arc<Node>,
 	paginated_kv_store: Arc<dyn PaginatedKVStore>,
 	api_key: String,
+	metrics: Arc<Metrics>,
 }
 
 impl NodeService {
 	pub(crate) fn new(
 		node: Arc<Node>, paginated_kv_store: Arc<dyn PaginatedKVStore>, api_key: String,
+		metrics: Arc<Metrics>,
 	) -> Self {
-		Self { node, paginated_kv_store, api_key }
+		Self { node, paginated_kv_store, api_key, metrics }
 	}
 }
 
@@ -154,6 +157,17 @@ impl Service<Request<Incoming>> for NodeService {
 	type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
 
 	fn call(&self, req: Request<Incoming>) -> Self::Future {
+		// Handle metrics endpoint separately to bypass auth and return plain text
+		if req.uri().path().len() > 1 && &req.uri().path()[1..] == GET_METRICS_PATH {
+			let metrics = Arc::clone(&self.metrics);
+			return Box::pin(async move {
+				Ok(Response::builder()
+					.header("Content-Type", "text/plain")
+					.body(Full::new(Bytes::from(metrics.gather_metrics())))
+					.unwrap())
+			});
+		}
+
 		// Extract auth params from headers (validation happens after body is read)
 		let auth_params = match extract_auth_params(&req) {
 			Ok(params) => params,