Add core health monitoring infrastructure for vmcp backends (#3100)

* Add core health monitoring infrastructure for vmcp backends

Implement health checking and status tracking for virtual MCP server
backends. This provides the foundation for monitoring backend availability
and categorizing failure modes (unhealthy, degraded, unauthenticated).

Related-to: #3036

* changes from review

* fixes from review

---------

Co-authored-by: taskbot <taskbot@users.noreply.github.com>

Author: yrobla

pkg/vmcp/client/client.go

@@ -7,8 +7,10 @@ package client
 import (
 	"context"
 	"encoding/base64"
+	"errors"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 
 	"github.com/mark3labs/mcp-go/client"
@@ -239,6 +241,69 @@ func (h *httpBackendClient) defaultClientFactory(ctx context.Context, target *vm
 	return c, nil
 }
 
+// wrapBackendError wraps an error with the appropriate sentinel error based on error type.
+// This enables type-safe error checking with errors.Is() instead of string matching.
+//
+// Error detection strategy (in order of preference):
+// 1. Check for standard Go error types (context errors, net.Error, url.Error)
+// 2. Fall back to string pattern matching for library-specific errors (MCP SDK, HTTP libs)
+//
+// Error chain preservation:
+// The returned error wraps the sentinel error (ErrTimeout, ErrBackendUnavailable, etc.) with %w
+// and formats the original error with %v. This means:
+// - errors.Is() works for checking the sentinel error (e.g., errors.Is(err, vmcp.ErrTimeout))
+// - errors.As() cannot access the underlying original error type
+// This is a deliberate trade-off due to Go's limitation of one %w per fmt.Errorf call.
+// If access to the underlying error type is needed in the future, consider implementing
+// a custom error type with multiple Unwrap() methods (Go 1.20+).
+func wrapBackendError(err error, backendID string, operation string) error {
+	if err == nil {
+		return nil
+	}
+
+	// 1. Type-based detection: Check for context deadline/cancellation
+	if errors.Is(err, context.DeadlineExceeded) {
+		return fmt.Errorf("%w: failed to %s for backend %s (timeout): %v",
+			vmcp.ErrTimeout, operation, backendID, err)
+	}
+	if errors.Is(err, context.Canceled) {
+		return fmt.Errorf("%w: failed to %s for backend %s (cancelled): %v",
+			vmcp.ErrCancelled, operation, backendID, err)
+	}
+
+	// 2. Type-based detection: Check for net.Error with Timeout() method
+	// This handles network timeouts from the standard library
+	var netErr net.Error
+	if errors.As(err, &netErr) && netErr.Timeout() {
+		return fmt.Errorf("%w: failed to %s for backend %s (timeout): %v",
+			vmcp.ErrTimeout, operation, backendID, err)
+	}
+
+	// 3. String-based detection: Fall back to pattern matching for cases where
+	// we don't have structured error types (MCP SDK, HTTP libraries with embedded status codes)
+	// Authentication errors (401, 403, auth failures)
+	if vmcp.IsAuthenticationError(err) {
+		return fmt.Errorf("%w: failed to %s for backend %s: %v",
+			vmcp.ErrAuthenticationFailed, operation, backendID, err)
+	}
+
+	// Timeout errors (deadline exceeded, timeout messages)
+	if vmcp.IsTimeoutError(err) {
+		return fmt.Errorf("%w: failed to %s for backend %s (timeout): %v",
+			vmcp.ErrTimeout, operation, backendID, err)
+	}
+
+	// Connection errors (refused, reset, unreachable)
+	if vmcp.IsConnectionError(err) {
+		return fmt.Errorf("%w: failed to %s for backend %s (connection error): %v",
+			vmcp.ErrBackendUnavailable, operation, backendID, err)
+	}
+
+	// Default to backend unavailable for unknown errors
+	return fmt.Errorf("%w: failed to %s for backend %s: %v",
+		vmcp.ErrBackendUnavailable, operation, backendID, err)
+}
+
 // initializeClient performs MCP protocol initialization handshake and returns server capabilities.
 // This allows the caller to determine which optional features the server supports.
 func initializeClient(ctx context.Context, c *client.Client) (*mcp.ServerCapabilities, error) {
@@ -313,14 +378,14 @@ func (h *httpBackendClient) ListCapabilities(ctx context.Context, target *vmcp.B
 	// Create a client for this backend (not yet initialized)
 	c, err := h.clientFactory(ctx, target)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create client for backend %s: %w", target.WorkloadID, err)
+		return nil, wrapBackendError(err, target.WorkloadID, "create client")
 	}
 	defer c.Close()
 
 	// Initialize the client and get server capabilities
 	serverCaps, err := initializeClient(ctx, c)
 	if err != nil {
-		return nil, fmt.Errorf("failed to initialize client for backend %s: %w", target.WorkloadID, err)
+		return nil, wrapBackendError(err, target.WorkloadID, "initialize client")
 	}
 
 	logger.Debugf("Backend %s capabilities: tools=%v, resources=%v, prompts=%v",
@@ -330,17 +395,17 @@ func (h *httpBackendClient) ListCapabilities(ctx context.Context, target *vmcp.B
 	// Check for nil BEFORE passing to functions to avoid interface{} nil pointer issues
 	toolsResp, err := queryTools(ctx, c, serverCaps.Tools != nil, target.WorkloadID)
 	if err != nil {
-		return nil, err
+		return nil, wrapBackendError(err, target.WorkloadID, "list tools")
 	}
 
 	resourcesResp, err := queryResources(ctx, c, serverCaps.Resources != nil, target.WorkloadID)
 	if err != nil {
-		return nil, err
+		return nil, wrapBackendError(err, target.WorkloadID, "list resources")
 	}
 
 	promptsResp, err := queryPrompts(ctx, c, serverCaps.Prompts != nil, target.WorkloadID)
 	if err != nil {
-		return nil, err
+		return nil, wrapBackendError(err, target.WorkloadID, "list prompts")
 	}
 
 	// Convert MCP types to vmcp types
@@ -428,13 +493,13 @@ func (h *httpBackendClient) CallTool(
 	// Create a client for this backend
 	c, err := h.clientFactory(ctx, target)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create client for backend %s: %w", target.WorkloadID, err)
+		return nil, wrapBackendError(err, target.WorkloadID, "create client")
 	}
 	defer c.Close()
 
 	// Initialize the client
 	if _, err := initializeClient(ctx, c); err != nil {
-		return nil, fmt.Errorf("failed to initialize client for backend %s: %w", target.WorkloadID, err)
+		return nil, wrapBackendError(err, target.WorkloadID, "initialize client")
 	}
 
 	// Call the tool using the original capability name from the backend's perspective.
@@ -525,13 +590,13 @@ func (h *httpBackendClient) ReadResource(ctx context.Context, target *vmcp.Backe
 	// Create a client for this backend
 	c, err := h.clientFactory(ctx, target)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create client for backend %s: %w", target.WorkloadID, err)
+		return nil, wrapBackendError(err, target.WorkloadID, "create client")
 	}
 	defer c.Close()
 
 	// Initialize the client
 	if _, err := initializeClient(ctx, c); err != nil {
-		return nil, fmt.Errorf("failed to initialize client for backend %s: %w", target.WorkloadID, err)
+		return nil, wrapBackendError(err, target.WorkloadID, "initialize client")
 	}
 
 	// Read the resource using the original URI from the backend's perspective.
@@ -586,13 +651,13 @@ func (h *httpBackendClient) GetPrompt(
 	// Create a client for this backend
 	c, err := h.clientFactory(ctx, target)
 	if err != nil {
-		return "", fmt.Errorf("failed to create client for backend %s: %w", target.WorkloadID, err)
+		return "", wrapBackendError(err, target.WorkloadID, "create client")
 	}
 	defer c.Close()
 
 	// Initialize the client
 	if _, err := initializeClient(ctx, c); err != nil {
-		return "", fmt.Errorf("failed to initialize client for backend %s: %w", target.WorkloadID, err)
+		return "", wrapBackendError(err, target.WorkloadID, "initialize client")
 	}
 
 	// Get the prompt using the original prompt name from the backend's perspective.

pkg/vmcp/errors.go

@@ -1,6 +1,9 @@
 package vmcp
 
-import "errors"
+import (
+	"errors"
+	"strings"
+)
 
 // Common domain errors used across vmcp subpackages.
 // Following DDD principles, domain errors are defined at the package root.
@@ -61,3 +64,82 @@ var (
 	// Wrapping errors should list the conflicting tool names.
 	ErrToolNameConflict = errors.New("tool name conflict")
 )
+
+// Error Categorization Helpers
+//
+// These functions categorize errors by examining error message strings.
+// They serve as a fallback mechanism for error detection when:
+//
+// 1. Errors come from external libraries that use their own error types and formats
+// 2. Legacy code paths don't wrap errors with sentinel errors
+// 3. Backwards compatibility is needed for error detection
+//
+// Note: BackendClient now wraps all errors with appropriate sentinel errors
+// (ErrAuthenticationFailed, ErrTimeout, ErrBackendUnavailable). Health monitoring
+// code should prefer errors.Is() checks over these string-based functions.
+// These functions remain for backwards compatibility and as a fallback mechanism.
+
+// IsAuthenticationError checks if an error message indicates an authentication failure.
+// Uses case-insensitive pattern matching to detect various auth error formats from
+// HTTP libraries, MCP protocol errors, and authentication middleware.
+func IsAuthenticationError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errLower := strings.ToLower(err.Error())
+
+	// Check for explicit authentication failure messages
+	if strings.Contains(errLower, "authentication failed") ||
+		strings.Contains(errLower, "authentication error") {
+		return true
+	}
+
+	// Check for HTTP 401/403 status codes with context
+	// Match patterns like "401 Unauthorized", "HTTP 401", "status code 401"
+	if strings.Contains(errLower, "401 unauthorized") ||
+		strings.Contains(errLower, "403 forbidden") ||
+		strings.Contains(errLower, "http 401") ||
+		strings.Contains(errLower, "http 403") ||
+		strings.Contains(errLower, "status code 401") ||
+		strings.Contains(errLower, "status code 403") {
+		return true
+	}
+
+	// Check for explicit unauthenticated/unauthorized errors
+	if strings.Contains(errLower, "request unauthenticated") ||
+		strings.Contains(errLower, "request unauthorized") ||
+		strings.Contains(errLower, "access denied") {
+		return true
+	}
+
+	return false
+}
+
+// IsTimeoutError checks if an error message indicates a timeout.
+// Detects various timeout formats from context deadlines, HTTP timeouts,
+// and network timeout errors.
+func IsTimeoutError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errLower := strings.ToLower(err.Error())
+	return strings.Contains(errLower, "timeout") ||
+		strings.Contains(errLower, "deadline exceeded") ||
+		strings.Contains(errLower, "context deadline exceeded")
+}
+
+// IsConnectionError checks if an error message indicates a connection failure.
+// Detects network-level errors like connection refused, reset, unreachable, etc.
+func IsConnectionError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errLower := strings.ToLower(err.Error())
+	return strings.Contains(errLower, "connection refused") ||
+		strings.Contains(errLower, "connection reset") ||
+		strings.Contains(errLower, "no route to host") ||
+		strings.Contains(errLower, "network is unreachable")
+}

pkg/vmcp/health/checker.go

@@ -0,0 +1,136 @@
+// Package health provides health monitoring for vMCP backend MCP servers.
+//
+// This package implements the HealthChecker interface and provides periodic
+// health monitoring with configurable intervals and failure thresholds.
+package health
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/stacklok/toolhive/pkg/logger"
+	"github.com/stacklok/toolhive/pkg/vmcp"
+)
+
+// healthChecker implements vmcp.HealthChecker using ListCapabilities as the health check.
+type healthChecker struct {
+	// client is the backend client used to communicate with backends.
+	client vmcp.BackendClient
+
+	// timeout is the timeout for health check operations.
+	timeout time.Duration
+
+	// degradedThreshold is the response time threshold for marking a backend as degraded.
+	// If a health check succeeds but takes longer than this duration, the backend is marked degraded.
+	// Zero means disabled (backends will never be marked degraded based on response time alone).
+	degradedThreshold time.Duration
+}
+
+// NewHealthChecker creates a new health checker that uses BackendClient.ListCapabilities
+// as the health check mechanism. This validates the full MCP communication stack:
+// network connectivity, MCP protocol compliance, authentication, and responsiveness.
+//
+// Parameters:
+//   - client: BackendClient for communicating with backend MCP servers
+//   - timeout: Maximum duration for health check operations (0 = no timeout)
+//   - degradedThreshold: Response time threshold for marking backend as degraded (0 = disabled)
+//
+// Returns a new HealthChecker implementation.
+func NewHealthChecker(client vmcp.BackendClient, timeout time.Duration, degradedThreshold time.Duration) vmcp.HealthChecker {
+	return &healthChecker{
+		client:            client,
+		timeout:           timeout,
+		degradedThreshold: degradedThreshold,
+	}
+}
+
+// CheckHealth performs a health check on a backend by calling ListCapabilities.
+// This validates the full MCP communication stack and returns the backend's health status.
+//
+// Health determination logic:
+//   - Success with fast response: Backend is healthy (BackendHealthy)
+//   - Success with slow response (> degradedThreshold): Backend is degraded (BackendDegraded)
+//   - Authentication error: Backend is unauthenticated (BackendUnauthenticated)
+//   - Timeout or connection error: Backend is unhealthy (BackendUnhealthy)
+//   - Other errors: Backend is unhealthy (BackendUnhealthy)
+//
+// The error return is informational and provides context about what failed.
+// The BackendHealthStatus return indicates the categorized health state.
+func (h *healthChecker) CheckHealth(ctx context.Context, target *vmcp.BackendTarget) (vmcp.BackendHealthStatus, error) {
+	// Apply timeout if configured
+	checkCtx := ctx
+	var cancel context.CancelFunc
+	if h.timeout > 0 {
+		checkCtx, cancel = context.WithTimeout(ctx, h.timeout)
+		defer cancel()
+	}
+
+	logger.Debugf("Performing health check for backend %s (%s)", target.WorkloadName, target.BaseURL)
+
+	// Track response time for degraded detection
+	startTime := time.Now()
+
+	// Use ListCapabilities as the health check - it performs:
+	// 1. Client creation with transport setup
+	// 2. MCP protocol initialization handshake
+	// 3. Capabilities query (tools, resources, prompts)
+	// This validates the full communication stack
+	_, err := h.client.ListCapabilities(checkCtx, target)
+	responseDuration := time.Since(startTime)
+
+	if err != nil {
+		// Categorize the error to determine health status
+		status := categorizeError(err)
+		logger.Debugf("Health check failed for backend %s: %v (status: %s, duration: %v)",
+			target.WorkloadName, err, status, responseDuration)
+		return status, fmt.Errorf("health check failed: %w", err)
+	}
+
+	// Check if response time indicates degraded performance
+	if h.degradedThreshold > 0 && responseDuration > h.degradedThreshold {
+		logger.Warnf("Health check succeeded for backend %s but response was slow: %v (threshold: %v) - marking as degraded",
+			target.WorkloadName, responseDuration, h.degradedThreshold)
+		return vmcp.BackendDegraded, nil
+	}
+
+	logger.Debugf("Health check succeeded for backend %s (duration: %v)", target.WorkloadName, responseDuration)
+	return vmcp.BackendHealthy, nil
+}
+
+// categorizeError determines the appropriate health status based on the error type.
+// This uses sentinel error checking with errors.Is() for type-safe error categorization.
+// Falls back to string-based detection for backwards compatibility with non-wrapped errors.
+func categorizeError(err error) vmcp.BackendHealthStatus {
+	if err == nil {
+		return vmcp.BackendHealthy
+	}
+
+	// 1. Type-safe detection: Check for sentinel errors using errors.Is()
+	// BackendClient now wraps all errors with appropriate sentinel errors
+	if errors.Is(err, vmcp.ErrAuthenticationFailed) || errors.Is(err, vmcp.ErrAuthorizationFailed) {
+		return vmcp.BackendUnauthenticated
+	}
+
+	if errors.Is(err, vmcp.ErrTimeout) || errors.Is(err, vmcp.ErrCancelled) {
+		return vmcp.BackendUnhealthy
+	}
+
+	if errors.Is(err, vmcp.ErrBackendUnavailable) {
+		return vmcp.BackendUnhealthy
+	}
+
+	// 2. String-based detection: Fallback for backwards compatibility
+	// This handles errors from sources that don't wrap with sentinel errors
+	if vmcp.IsAuthenticationError(err) {
+		return vmcp.BackendUnauthenticated
+	}
+
+	if vmcp.IsTimeoutError(err) || vmcp.IsConnectionError(err) {
+		return vmcp.BackendUnhealthy
+	}
+
+	// Default to unhealthy for unknown errors
+	return vmcp.BackendUnhealthy
+}