integration test for metrics

Signed-off-by: Jeremy Drouillard <jeremy@stacklok.com>

Author: jerm-dro

pkg/vmcp/server/server.go

@@ -199,6 +199,17 @@ func New(
 	// This provides SDK-agnostic elicitation with security validation
 	elicitationHandler := composer.NewDefaultElicitationHandler(sdkElicitationRequester)
 
+	// Decorate backend client with telemetry if provider is configured
+	// This must happen BEFORE creating the workflow engine so that workflow
+	// backend calls are instrumented when they occur during workflow execution.
+	if cfg.TelemetryProvider != nil {
+		var err error
+		backendClient, err = monitorBackends(context.Background(), cfg.TelemetryProvider.MeterProvider(), backends, backendClient)
+		if err != nil {
+			return nil, fmt.Errorf("failed to monitor backends: %w", err)
+		}
+	}
+
 	// Create workflow engine (composer) for executing composite tools
 	// The composer orchestrates multi-step workflows across backends
 	// Use in-memory state store with 5-minute cleanup interval and 1-hour max age for completed workflows
@@ -211,12 +222,8 @@ func New(
 		return nil, fmt.Errorf("workflow validation failed: %w", err)
 	}
 
-	// Decorate backend client and workflow executors with telemetry if provider is configured
+	// Decorate workflow executors with telemetry if provider is configured
 	if cfg.TelemetryProvider != nil {
-		backendClient, err = monitorBackends(context.Background(), cfg.TelemetryProvider.MeterProvider(), backends, backendClient)
-		if err != nil {
-			return nil, fmt.Errorf("failed to monitor backends: %w", err)
-		}
 		workflowExecutors, err = monitorWorkflowExecutors(cfg.TelemetryProvider.MeterProvider(), workflowExecutors)
 		if err != nil {
 			return nil, fmt.Errorf("failed to monitor workflow executors: %w", err)

pkg/vmcp/server/telemetry.go

@@ -30,15 +30,15 @@ func monitorBackends(ctx context.Context, meterProvider metric.MeterProvider, ba
 	}
 	backendCount.Record(ctx, int64(len(backends)))
 
-	requestsTotal, err := meter.Int64Counter("toolhive_vmcp_requests_total", metric.WithDescription("Total number of requests per backend"))
+	requestsTotal, err := meter.Int64Counter("toolhive_vmcp_backend_requests_total", metric.WithDescription("Total number of requests per backend"))
 	if err != nil {
 		return nil, fmt.Errorf("failed to create requests total counter: %w", err)
 	}
-	errorsTotal, err := meter.Int64Counter("toolhive_vmcp_errors_total", metric.WithDescription("Total number of errors per backend"))
+	errorsTotal, err := meter.Int64Counter("toolhive_vmcp_backend_errors_total", metric.WithDescription("Total number of errors per backend"))
 	if err != nil {
 		return nil, fmt.Errorf("failed to create errors total counter: %w", err)
 	}
-	requestsDuration, err := meter.Float64Histogram("toolhive_vmcp_requests_duration", metric.WithDescription("Duration of requests in seconds per backend"))
+	requestsDuration, err := meter.Float64Histogram("toolhive_vmcp_backend_requests_duration", metric.WithDescription("Duration of requests in seconds per backend"))
 	if err != nil {
 		return nil, fmt.Errorf("failed to create requests duration histogram: %w", err)
 	}

test/integration/vmcp/helpers/vmcp_server.go

@@ -10,11 +10,13 @@ import (
 
 	"github.com/stacklok/toolhive/pkg/auth"
 	"github.com/stacklok/toolhive/pkg/env"
+	"github.com/stacklok/toolhive/pkg/telemetry"
 	vmcptypes "github.com/stacklok/toolhive/pkg/vmcp"
 	"github.com/stacklok/toolhive/pkg/vmcp/aggregator"
 	"github.com/stacklok/toolhive/pkg/vmcp/auth/factory"
 	authtypes "github.com/stacklok/toolhive/pkg/vmcp/auth/types"
 	vmcpclient "github.com/stacklok/toolhive/pkg/vmcp/client"
+	"github.com/stacklok/toolhive/pkg/vmcp/composer"
 	"github.com/stacklok/toolhive/pkg/vmcp/discovery"
 	"github.com/stacklok/toolhive/pkg/vmcp/router"
 	vmcpserver "github.com/stacklok/toolhive/pkg/vmcp/server"
@@ -63,8 +65,10 @@ type VMCPServerOption func(*vmcpServerConfig)
 
 // vmcpServerConfig holds configuration for creating a test vMCP server.
 type vmcpServerConfig struct {
-	conflictStrategy string
-	prefixFormat     string
+	conflictStrategy  string
+	prefixFormat      string
+	workflowDefs      map[string]*composer.WorkflowDefinition
+	telemetryProvider *telemetry.Provider
 }
 
 // WithPrefixConflictResolution configures prefix-based conflict resolution.
@@ -75,6 +79,20 @@ func WithPrefixConflictResolution(format string) VMCPServerOption {
 	}
 }
 
+// WithWorkflowDefinitions configures composite tool workflow definitions.
+func WithWorkflowDefinitions(defs map[string]*composer.WorkflowDefinition) VMCPServerOption {
+	return func(c *vmcpServerConfig) {
+		c.workflowDefs = defs
+	}
+}
+
+// WithTelemetryProvider configures the telemetry provider.
+func WithTelemetryProvider(provider *telemetry.Provider) VMCPServerOption {
+	return func(c *vmcpServerConfig) {
+		c.telemetryProvider = provider
+	}
+}
+
 // getFreePort returns an available TCP port on localhost.
 // This is used for parallel test execution to avoid port conflicts.
 func getFreePort(tb testing.TB) int {
@@ -148,12 +166,13 @@ func NewVMCPServer(
 
 	// Create vMCP server with test-specific defaults
 	vmcpServer, err := vmcpserver.New(&vmcpserver.Config{
-		Name:           "test-vmcp",
-		Version:        "1.0.0",
-		Host:           "127.0.0.1",
-		Port:           getFreePort(tb), // Get a random available port for parallel test execution
-		AuthMiddleware: auth.AnonymousMiddleware,
-	}, rtr, backendClient, discoveryMgr, backends, nil) // nil for workflowDefs in tests
+		Name:              "test-vmcp",
+		Version:           "1.0.0",
+		Host:              "127.0.0.1",
+		Port:              getFreePort(tb), // Get a random available port for parallel test execution
+		AuthMiddleware:    auth.AnonymousMiddleware,
+		TelemetryProvider: config.telemetryProvider,
+	}, rtr, backendClient, discoveryMgr, backends, config.workflowDefs)
 	require.NoError(tb, err, "failed to create vMCP server")
 
 	// Start server automatically

test/integration/vmcp/vmcp_integration_test.go

@@ -2,12 +2,19 @@ package vmcp_test
 
 import (
 	"context"
+	"io"
+	"net/http"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
+	"github.com/stacklok/toolhive/pkg/telemetry"
 	"github.com/stacklok/toolhive/pkg/vmcp"
 	authtypes "github.com/stacklok/toolhive/pkg/vmcp/auth/types"
+	"github.com/stacklok/toolhive/pkg/vmcp/composer"
 	"github.com/stacklok/toolhive/test/integration/vmcp/helpers"
 )
 
@@ -225,3 +232,123 @@ func TestVMCPServer_TwoBoundaryAuth_HeaderInjection(t *testing.T) {
 		helpers.AssertTextNotContains(t, text, "error", "failed", "leakage")
 	})
 }
+
+// TestVMCPServer_Telemetry_CompositeToolMetrics verifies that vMCP exposes
+// Prometheus metrics for composite tool workflow executions and backend requests on /metrics.
+// This test creates a composite tool, executes it, and verifies the metrics
+// for both the workflow and the backend subtool calls are correctly exposed.
+func TestVMCPServer_Telemetry_CompositeToolMetrics(t *testing.T) {
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Setup: Create a synthetic MCP backend server with a simple tool
+	echoServer := helpers.CreateBackendServer(t, []helpers.BackendTool{
+		helpers.NewBackendTool("echo", "Echo the input message",
+			func(_ context.Context, args map[string]any) string {
+				msg, _ := args["message"].(string)
+				return `{"echoed": "` + msg + `"}`
+			}),
+	}, helpers.WithBackendName("echo-mcp"))
+	defer echoServer.Close()
+
+	// Configure backend pointing to test server
+	backends := []vmcp.Backend{
+		helpers.NewBackend("echo",
+			helpers.WithURL(echoServer.URL+"/mcp"),
+			helpers.WithMetadata("group", "test-group"),
+		),
+	}
+
+	// Create composite tool workflow definition that calls the echo tool
+	workflowDefs := map[string]*composer.WorkflowDefinition{
+		"echo_workflow": {
+			Name:        "echo_workflow",
+			Description: "A composite tool that echoes a message",
+			Parameters: map[string]any{
+				"type": "object",
+				"properties": map[string]any{
+					"message": map[string]any{
+						"type":        "string",
+						"description": "The message to echo",
+					},
+				},
+				"required": []string{"message"},
+			},
+			Steps: []composer.WorkflowStep{
+				{
+					ID:   "echo_step",
+					Type: "tool",
+					Tool: "echo_echo", // prefixed with backend name
+					Arguments: map[string]any{
+						"message": "{{.params.message}}",
+					},
+				},
+			},
+			Timeout: 30 * time.Second,
+		},
+	}
+
+	// Create telemetry provider with Prometheus enabled
+	telemetryConfig := telemetry.Config{
+		ServiceName:                 "vmcp-telemetry-test",
+		ServiceVersion:              "1.0.0",
+		EnablePrometheusMetricsPath: true,
+	}
+	telemetryProvider, err := telemetry.NewProvider(ctx, telemetryConfig)
+	require.NoError(t, err, "failed to create telemetry provider")
+	defer telemetryProvider.Shutdown(ctx)
+
+	// Create vMCP server with composite tool and telemetry
+	vmcpServer := helpers.NewVMCPServer(ctx, t, backends,
+		helpers.WithPrefixConflictResolution("{workload}_"),
+		helpers.WithWorkflowDefinitions(workflowDefs),
+		helpers.WithTelemetryProvider(telemetryProvider),
+	)
+
+	// Create and initialize MCP client
+	vmcpURL := "http://" + vmcpServer.Address() + "/mcp"
+	client := helpers.NewMCPClient(ctx, t, vmcpURL)
+	defer client.Close()
+
+	// Call the composite tool
+	resp := client.CallTool(ctx, "echo_workflow", map[string]any{"message": "hello world"})
+	text := helpers.AssertToolCallSuccess(t, resp)
+	helpers.AssertTextContains(t, text, "echoed", "hello world")
+
+	// Fetch metrics from /metrics endpoint
+	metricsURL := "http://" + vmcpServer.Address() + "/metrics"
+	httpClient := &http.Client{Timeout: 5 * time.Second}
+	metricsResp, err := httpClient.Get(metricsURL)
+	require.NoError(t, err, "failed to fetch metrics")
+	defer metricsResp.Body.Close()
+
+	require.Equal(t, http.StatusOK, metricsResp.StatusCode, "metrics endpoint should return 200")
+
+	body, err := io.ReadAll(metricsResp.Body)
+	require.NoError(t, err, "failed to read metrics body")
+	metricsContent := string(body)
+
+	// Log metrics for debugging
+	t.Logf("Metrics content:\n%s", metricsContent)
+
+	// Verify workflow execution metrics are present (composite tool).
+	assert.True(t, strings.Contains(metricsContent, "toolhive_vmcp_workflow_executions_total"),
+		"Should contain workflow executions total metric")
+	assert.True(t, strings.Contains(metricsContent, "toolhive_vmcp_workflow_duration_seconds"),
+		"Should contain workflow duration metric")
+	assert.True(t, strings.Contains(metricsContent, `workflow_name="echo_workflow"`),
+		"Should contain workflow name label")
+
+	// Verify backend metrics are present.
+	assert.True(t, strings.Contains(metricsContent, "toolhive_vmcp_backend_requests_total"),
+		"Should contain backend requests total metric")
+	assert.True(t, strings.Contains(metricsContent, "toolhive_vmcp_backend_requests_duration"),
+		"Should contain backend requests duration metric")
+
+	// Verify HTTP middleware metrics are present (incoming MCP requests).
+	assert.True(t, strings.Contains(metricsContent, "toolhive_mcp_requests_total"),
+		"Should contain HTTP middleware requests total metric")
+	assert.True(t, strings.Contains(metricsContent, "toolhive_mcp_request_duration_seconds"),
+		"Should contain HTTP middleware request duration metric")
+}