fix: ext_proc-driven multi-provider routing, weight=0, user-agent metering (opendatahub-io#399)

- model-provider-resolver: set x-ipp-selected-provider header using
  provider CR name (providerName field) after selectByWeight
- selectByWeight: skip refs with weight <= 0 (disabled providers)
- CRD: allow weight=0 (Minimum changed from 1 to 0)
- HTTPRoute reconciler: per-provider route rules with header match on
  x-ipp-selected-provider, fallback to refs[0]
- external-metering: capture User-Agent from request headers for client
  attribution in usage and error events

Author: noyitz

api/inference/v1alpha1/externalmodel_types.go

@@ -100,9 +100,10 @@ type ExternalProviderRef struct {
 
 	// Weight determines the relative traffic proportion for this provider binding.
 	// Higher weight means more traffic. Used for weighted random selection across
-	// multiple provider refs. Defaults to 1 if not set.
+	// multiple provider refs. A weight of 0 disables the ref (no traffic routed
+	// to it). Defaults to 1 if not set.
 	// +optional
-	// +kubebuilder:validation:Minimum=1
+	// +kubebuilder:validation:Minimum=0
 	// +kubebuilder:validation:Maximum=100
 	// +kubebuilder:default=1
 	Weight *int `json:"weight,omitempty"`

config/crd/bases/inference.opendatahub.io_externalmodels.yaml

@@ -139,9 +139,10 @@ spec:
                       description: |-
                         Weight determines the relative traffic proportion for this provider binding.
                         Higher weight means more traffic. Used for weighted random selection across
-                        multiple provider refs. Defaults to 1 if not set.
+                        multiple provider refs. A weight of 0 disables the ref (no traffic routed
+                        to it). Defaults to 1 if not set.
                       maximum: 100
-                      minimum: 1
+                      minimum: 0
                       type: integer
                   required:
                   - apiFormat

pkg/controller/externalmodel/reconciler.go

@@ -19,6 +19,7 @@ package externalmodel
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/go-logr/logr"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -111,33 +112,55 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	return ctrl.Result{}, nil
 }
 
+// resolvedRef holds the ExternalProvider info resolved for one ExternalProviderRef.
+type resolvedRef struct {
+	providerName     string
+	providerEndpoint string
+	targetModel      string
+}
+
 func (r *Reconciler) reconcileHTTPRoute(ctx context.Context, logger logr.Logger, model *inferencev1alpha1.ExternalModel) error {
 	if len(model.Spec.ExternalProviderRefs) == 0 {
 		return fmt.Errorf("ExternalModel %q has no externalProviderRefs", model.Name)
 	}
-	ref := model.Spec.ExternalProviderRefs[0]
 
-	provider := &inferencev1alpha1.ExternalProvider{}
-	providerKey := types.NamespacedName{Name: ref.Ref.Name, Namespace: model.Namespace}
-	if err := r.Get(ctx, providerKey, provider); err != nil {
-		if apierrors.IsNotFound(err) {
-			return fmt.Errorf("ExternalProvider %q not found in namespace %q", ref.Ref.Name, model.Namespace)
+	var resolved []resolvedRef
+	var skipReasons []string
+	for _, ref := range model.Spec.ExternalProviderRefs {
+		provider := &inferencev1alpha1.ExternalProvider{}
+		providerKey := types.NamespacedName{Name: ref.Ref.Name, Namespace: model.Namespace}
+		if err := r.Get(ctx, providerKey, provider); err != nil {
+			if apierrors.IsNotFound(err) {
+				logger.Error(err, "ExternalProvider not found, skipping", "provider", ref.Ref.Name)
+				skipReasons = append(skipReasons, fmt.Sprintf("ExternalProvider %q not found in namespace %q", ref.Ref.Name, model.Namespace))
+				continue
+			}
+			return fmt.Errorf("failed to get ExternalProvider %q: %w", ref.Ref.Name, err)
 		}
-		return fmt.Errorf("failed to get ExternalProvider %q: %w", ref.Ref.Name, err)
-	}
-
-	if provider.Status.Phase != "Ready" {
-		return fmt.Errorf("ExternalProvider %q is not ready (phase: %s)", ref.Ref.Name, provider.Status.Phase)
+		if provider.Status.Phase != "Ready" {
+			logger.Info("ExternalProvider not ready, skipping", "provider", ref.Ref.Name, "phase", provider.Status.Phase)
+			skipReasons = append(skipReasons, fmt.Sprintf("ExternalProvider %q is not ready (phase: %s)", ref.Ref.Name, provider.Status.Phase))
+			continue
+		}
+		if _, err := ctrlcommon.ResolvePath(ref.Path, mergeConfig(provider.Spec.Config, ref.Config), ref.TargetModel); err != nil {
+			logger.Error(err, "path resolution failed, skipping", "provider", ref.Ref.Name, "path", ref.Path)
+			skipReasons = append(skipReasons, fmt.Sprintf("path %q: %v", ref.Path, err))
+			continue
+		}
+		resolved = append(resolved, resolvedRef{
+			providerName:     provider.Name,
+			providerEndpoint: provider.Spec.Endpoint,
+			targetModel:      ref.TargetModel,
+		})
 	}
 
-	if _, err := ctrlcommon.ResolvePath(ref.Path, mergeConfig(provider.Spec.Config, ref.Config), ref.TargetModel); err != nil {
-		return fmt.Errorf("path %q: %w", ref.Path, err)
+	if len(resolved) == 0 {
+		return fmt.Errorf("ExternalModel %q: no provider refs resolved successfully: %s", model.Name, strings.Join(skipReasons, "; "))
 	}
 
 	labels := commonLabels(model.Name)
 	hr := buildHTTPRoute(
-		provider.Spec.Endpoint,
-		provider.Name,
+		resolved,
 		model.Name,
 		model.Namespace,
 		ctrlcommon.DefaultTLSPort,
@@ -155,10 +178,14 @@ func (r *Reconciler) reconcileHTTPRoute(ctx context.Context, logger logr.Logger,
 		return fmt.Errorf("failed to apply HTTPRoute: %w", err)
 	}
 
+	providerNames := make([]string, len(resolved))
+	for i, rr := range resolved {
+		providerNames[i] = rr.providerName
+	}
 	logger.Info("ExternalModel HTTPRoute reconciled",
 		"httpRoute", model.Name,
-		"provider", provider.Name,
-		"targetModel", ref.TargetModel,
+		"providers", providerNames,
+		"targetModel", resolved[0].targetModel,
 	)
 	return nil
 }
@@ -242,39 +269,102 @@ func commonLabels(modelName string) map[string]string {
 	}
 }
 
-func buildHTTPRoute(providerEndpoint, providerName, modelName, namespace string, port int32, gatewayName, gatewayNamespace, routeTimeout string, labels map[string]string) *gatewayapiv1.HTTPRoute {
+// selectedProviderHeader is the header the ext_proc sets to drive routing.
+const selectedProviderHeader = "x-ipp-selected-provider"
+
+func buildHTTPRoute(refs []resolvedRef, modelName, namespace string, port int32, gatewayName, gatewayNamespace, routeTimeout string, labels map[string]string) *gatewayapiv1.HTTPRoute {
 	gwNamespace := gatewayapiv1.Namespace(gatewayNamespace)
 	pathType := gatewayapiv1.PathMatchPathPrefix
 	pathPrefix := "/" + namespace + "/" + modelName
 	headerType := gatewayapiv1.HeaderMatchExact
 	gwPort := gatewayapiv1.PortNumber(port)
 	timeout := gatewayapiv1.Duration(routeTimeout)
 
-	backendRefs := []gatewayapiv1.HTTPBackendRef{
-		{
+	var rules []gatewayapiv1.HTTPRouteRule
+
+	for _, ref := range refs {
+		backendRefs := []gatewayapiv1.HTTPBackendRef{{
 			BackendRef: gatewayapiv1.BackendRef{
 				BackendObjectReference: gatewayapiv1.BackendObjectReference{
-					Name: gatewayapiv1.ObjectName(providerName),
+					Name: gatewayapiv1.ObjectName(ref.providerName),
 					Port: &gwPort,
 				},
 			},
-		},
-	}
-
-	filters := []gatewayapiv1.HTTPRouteFilter{
-		{
+		}}
+		filters := []gatewayapiv1.HTTPRouteFilter{{
 			Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier,
 			RequestHeaderModifier: &gatewayapiv1.HTTPHeaderFilter{
 				Set: []gatewayapiv1.HTTPHeader{
-					{
-						Name:  "Host",
-						Value: providerEndpoint,
-					},
+					{Name: "Host", Value: ref.providerEndpoint},
 				},
 			},
-		},
+		}}
+
+		rules = append(rules, gatewayapiv1.HTTPRouteRule{
+			Matches: []gatewayapiv1.HTTPRouteMatch{{
+				Path: &gatewayapiv1.HTTPPathMatch{Type: &pathType, Value: &pathPrefix},
+				Headers: []gatewayapiv1.HTTPHeaderMatch{{
+					Name: selectedProviderHeader, Type: &headerType, Value: ref.providerName,
+				}},
+			}},
+			BackendRefs: backendRefs,
+			Filters:     filters,
+			Timeouts:    &gatewayapiv1.HTTPRouteTimeouts{Request: &timeout},
+		})
+
+		rules = append(rules, gatewayapiv1.HTTPRouteRule{
+			Matches: []gatewayapiv1.HTTPRouteMatch{{
+				Path: &gatewayapiv1.HTTPPathMatch{Type: &pathType, Value: func() *string { s := "/"; return &s }()},
+				Headers: []gatewayapiv1.HTTPHeaderMatch{
+					{Name: "X-Gateway-Model-Name", Type: &headerType, Value: ref.targetModel},
+					{Name: selectedProviderHeader, Type: &headerType, Value: ref.providerName},
+				},
+			}},
+			BackendRefs: backendRefs,
+			Filters:     filters,
+			Timeouts:    &gatewayapiv1.HTTPRouteTimeouts{Request: &timeout},
+		})
 	}
 
+	// Fallback rules (no x-ipp-selected-provider): route to refs[0]
+	fallbackBackendRefs := []gatewayapiv1.HTTPBackendRef{{
+		BackendRef: gatewayapiv1.BackendRef{
+			BackendObjectReference: gatewayapiv1.BackendObjectReference{
+				Name: gatewayapiv1.ObjectName(refs[0].providerName),
+				Port: &gwPort,
+			},
+		},
+	}}
+	fallbackFilters := []gatewayapiv1.HTTPRouteFilter{{
+		Type: gatewayapiv1.HTTPRouteFilterRequestHeaderModifier,
+		RequestHeaderModifier: &gatewayapiv1.HTTPHeaderFilter{
+			Set: []gatewayapiv1.HTTPHeader{
+				{Name: "Host", Value: refs[0].providerEndpoint},
+			},
+		},
+	}}
+
+	rules = append(rules,
+		gatewayapiv1.HTTPRouteRule{
+			Matches: []gatewayapiv1.HTTPRouteMatch{{
+				Path: &gatewayapiv1.HTTPPathMatch{Type: &pathType, Value: &pathPrefix},
+			}},
+			BackendRefs: fallbackBackendRefs,
+			Filters:     fallbackFilters,
+			Timeouts:    &gatewayapiv1.HTTPRouteTimeouts{Request: &timeout},
+		},
+		gatewayapiv1.HTTPRouteRule{
+			Matches: []gatewayapiv1.HTTPRouteMatch{{
+				Headers: []gatewayapiv1.HTTPHeaderMatch{
+					{Name: "X-Gateway-Model-Name", Type: &headerType, Value: refs[0].targetModel},
+				},
+			}},
+			BackendRefs: fallbackBackendRefs,
+			Filters:     fallbackFilters,
+			Timeouts:    &gatewayapiv1.HTTPRouteTimeouts{Request: &timeout},
+		},
+	)
+
 	return &gatewayapiv1.HTTPRoute{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      modelName,
@@ -283,45 +373,12 @@ func buildHTTPRoute(providerEndpoint, providerName, modelName, namespace string,
 		},
 		Spec: gatewayapiv1.HTTPRouteSpec{
 			CommonRouteSpec: gatewayapiv1.CommonRouteSpec{
-				ParentRefs: []gatewayapiv1.ParentReference{
-					{
-						Name:      gatewayapiv1.ObjectName(gatewayName),
-						Namespace: &gwNamespace,
-					},
-				},
-			},
-			Rules: []gatewayapiv1.HTTPRouteRule{
-				// TODO: remove path prefix rule when unified entrypoint (RHAISTRAT-1540) is wired.
-				{
-					Matches: []gatewayapiv1.HTTPRouteMatch{
-						{
-							Path: &gatewayapiv1.HTTPPathMatch{
-								Type:  &pathType,
-								Value: &pathPrefix,
-							},
-						},
-					},
-					BackendRefs: backendRefs,
-					Filters:     filters,
-					Timeouts:    &gatewayapiv1.HTTPRouteTimeouts{Request: &timeout},
-				},
-				{
-					Matches: []gatewayapiv1.HTTPRouteMatch{
-						{
-							Headers: []gatewayapiv1.HTTPHeaderMatch{
-								{
-									Name:  "X-Gateway-Model-Name",
-									Type:  &headerType,
-									Value: modelName,
-								},
-							},
-						},
-					},
-					BackendRefs: backendRefs,
-					Filters:     filters,
-					Timeouts:    &gatewayapiv1.HTTPRouteTimeouts{Request: &timeout},
-				},
+				ParentRefs: []gatewayapiv1.ParentReference{{
+					Name:      gatewayapiv1.ObjectName(gatewayName),
+					Namespace: &gwNamespace,
+				}},
 			},
+			Rules: rules,
 		},
 	}
 }

pkg/controller/externalmodel/reconciler_test.go

@@ -360,72 +360,53 @@ func TestCommonLabels(t *testing.T) {
 	assert.Len(t, labels, 2)
 }
 
-func TestBuildHTTPRoute(t *testing.T) {
-	hr := buildHTTPRoute(
-		"api.openai.com", "my-openai",
-		"gpt4",
-		"models", 443,
-		"default-gateway", "openshift-ingress", "300s",
-		commonLabels("gpt4"),
-	)
+func TestBuildHTTPRoute_SingleProvider(t *testing.T) {
+	refs := []resolvedRef{{
+		providerName: "my-openai", providerEndpoint: "api.openai.com", targetModel: "gpt-4o",
+	}}
+	hr := buildHTTPRoute(refs, "gpt4", "models", 443,
+		"default-gateway", "openshift-ingress", "300s", commonLabels("gpt4"))
 
 	assert.Equal(t, "gpt4", hr.Name)
 	assert.Equal(t, "models", hr.Namespace)
-	assert.Equal(t, managedByValue, hr.Labels[ctrlcommon.LabelManagedBy])
-
-	// Parent gateway ref
-	require.Len(t, hr.Spec.ParentRefs, 1)
-	assert.Equal(t, "default-gateway", string(hr.Spec.ParentRefs[0].Name))
-	assert.Equal(t, "openshift-ingress", string(*hr.Spec.ParentRefs[0].Namespace))
-
-	// Must have 2 rules: path-based and header-based
-	require.Len(t, hr.Spec.Rules, 2)
 
-	// Rule 1: path-based match with namespace prefix
-	rule1 := hr.Spec.Rules[0]
-	assert.Equal(t, "/models/gpt4", *rule1.Matches[0].Path.Value)
+	// 4 rules: 2 per-provider + 2 fallback
+	require.Len(t, hr.Spec.Rules, 4)
 
-	// Rule 2: header-based match uses modelName (what /v1/models returns)
-	rule2 := hr.Spec.Rules[1]
-	assert.Equal(t, "X-Gateway-Model-Name", string(rule2.Matches[0].Headers[0].Name))
-	assert.Equal(t, "gpt4", rule2.Matches[0].Headers[0].Value)
+	// Rule 0: path + x-ipp-selected-provider
+	assert.Equal(t, "/models/gpt4", *hr.Spec.Rules[0].Matches[0].Path.Value)
+	require.Len(t, hr.Spec.Rules[0].Matches[0].Headers, 1)
+	assert.Equal(t, "x-ipp-selected-provider", string(hr.Spec.Rules[0].Matches[0].Headers[0].Name))
+	assert.Equal(t, "my-openai", hr.Spec.Rules[0].Matches[0].Headers[0].Value)
+	assert.Equal(t, "my-openai", string(hr.Spec.Rules[0].BackendRefs[0].Name))
 
-	// Backend ref points to the PROVIDER's Service, not the model
-	for i, rule := range hr.Spec.Rules {
-		require.Len(t, rule.BackendRefs, 1, "rule %d", i)
-		assert.Equal(t, "my-openai", string(rule.BackendRefs[0].Name),
-			"rule %d: backend should be the provider's Service", i)
-	}
+	// Rule 2: fallback path (no provider header)
+	assert.Empty(t, hr.Spec.Rules[2].Matches[0].Headers)
 
-	// Host header filter for TLS SNI uses provider endpoint
-	for i, rule := range hr.Spec.Rules {
-		require.Len(t, rule.Filters, 1, "rule %d", i)
-		assert.Equal(t, gatewayapiv1.HTTPRouteFilterRequestHeaderModifier, rule.Filters[0].Type)
-		assert.Equal(t, "Host", string(rule.Filters[0].RequestHeaderModifier.Set[0].Name))
-		assert.Equal(t, "api.openai.com", rule.Filters[0].RequestHeaderModifier.Set[0].Value)
-	}
+	// Host filter
+	assert.Equal(t, "api.openai.com", hr.Spec.Rules[0].Filters[0].RequestHeaderModifier.Set[0].Value)
 }
 
-func TestBuildHTTPRoute_TargetModelDiffersFromName(t *testing.T) {
-	hr := buildHTTPRoute(
-		"bedrock.us-east-1.amazonaws.com", "my-bedrock",
-		"claude",
-		"models", 443,
-		"my-gateway", "gateway-ns", "300s",
-		commonLabels("claude"),
-	)
+func TestBuildHTTPRoute_MultiProvider(t *testing.T) {
+	refs := []resolvedRef{
+		{providerName: "anthropic", providerEndpoint: "api.anthropic.com", targetModel: "claude-opus-4-8"},
+		{providerName: "sim-vertex", providerEndpoint: "sim.example.com", targetModel: "claude-opus-4-8"},
+	}
+	hr := buildHTTPRoute(refs, "claude", "models", 443,
+		"my-gateway", "gateway-ns", "300s", commonLabels("claude"))
 
-	// Name and path use ExternalModel name
-	assert.Equal(t, "claude", hr.Name)
-	assert.Equal(t, "/models/claude", *hr.Spec.Rules[0].Matches[0].Path.Value)
+	// 6 rules: 2 per-provider x 2 + 2 fallback
+	require.Len(t, hr.Spec.Rules, 6)
 
-	// Header match uses modelName (what /v1/models returns to clients)
-	assert.Equal(t, "claude", hr.Spec.Rules[1].Matches[0].Headers[0].Value)
+	// Anthropic rules (0,1)
+	assert.Equal(t, "anthropic", hr.Spec.Rules[0].Matches[0].Headers[0].Value)
+	assert.Equal(t, "anthropic", string(hr.Spec.Rules[0].BackendRefs[0].Name))
 
-	// Backend points to provider Service
-	assert.Equal(t, "my-bedrock", string(hr.Spec.Rules[0].BackendRefs[0].Name))
+	// Sim-vertex rules (2,3)
+	assert.Equal(t, "sim-vertex", hr.Spec.Rules[2].Matches[0].Headers[0].Value)
+	assert.Equal(t, "sim-vertex", string(hr.Spec.Rules[2].BackendRefs[0].Name))
 
-	// Host header uses provider endpoint
-	assert.Equal(t, "bedrock.us-east-1.amazonaws.com",
-		hr.Spec.Rules[0].Filters[0].RequestHeaderModifier.Set[0].Value)
+	// Fallback rules (4,5) point to refs[0] (anthropic)
+	assert.Empty(t, hr.Spec.Rules[4].Matches[0].Headers)
+	assert.Equal(t, "anthropic", string(hr.Spec.Rules[4].BackendRefs[0].Name))
 }