diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 565fb1d034..b3df680d26 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -109,5 +109,6 @@ jobs: PROVIDER_TYPE: ${{ matrix.provider_type }} TEST_LABEL: ${{ matrix.cases_subset }} TEST_ENV: CI + E2E_EXEC_ADC_TIMEOUT: "30s" run: | make benchmark-test diff --git a/Makefile b/Makefile index 4249a3a432..4a61780048 100644 --- a/Makefile +++ b/Makefile @@ -185,9 +185,10 @@ kind-down: .PHONY: kind-load-images kind-load-images: pull-infra-images kind-load-ingress-image kind-load-adc-image - @kind load docker-image kennethreitz/httpbin:latest --name $(KIND_NAME) + @kind load docker-image kennethreitz/httpbin:latest --name $(KIND_NAME) @kind load docker-image jmalloc/echo-server:latest --name $(KIND_NAME) @kind load docker-image openresty/openresty:1.27.1.2-4-bullseye-fat --name $(KIND_NAME) + @kind load docker-image alpine/curl:8.17.0 --name $(KIND_NAME) .PHONY: kind-load-ingress-image kind-load-ingress-image: @@ -204,6 +205,7 @@ pull-infra-images: @docker pull kennethreitz/httpbin:latest @docker pull jmalloc/echo-server:latest @docker pull openresty/openresty:1.27.1.2-4-bullseye-fat + @docker pull alpine/curl:8.17.0 ##@ Build diff --git a/config/samples/config.yaml b/config/samples/config.yaml index 8e37192257..b9b9384b2f 100644 --- a/config/samples/config.yaml +++ b/config/samples/config.yaml @@ -34,6 +34,11 @@ exec_adc_timeout: 15s # The timeout for the ADC to execute. # The default value is 15 seconds. disable_gateway_api: false # Whether to disable the Gateway API support. # The default value is false. +listener_port_match_mode: "auto" # Mode for injecting server_port route vars from Gateway listener ports. + # - "auto": inject when parentRefs explicitly target listeners (sectionName/port) or when multiple listener ports are matched. + # - "explicit": inject only when parentRefs explicitly target listeners. + # - "off": never inject server_port vars. + # The default value is "auto". provider: type: "apisix" # Provider type. diff --git a/docs/en/latest/concepts/gateway-api.md b/docs/en/latest/concepts/gateway-api.md index cefdde190e..9b8ad48d50 100644 --- a/docs/en/latest/concepts/gateway-api.md +++ b/docs/en/latest/concepts/gateway-api.md @@ -78,7 +78,7 @@ The fields below are specified in the Gateway API specification but are either p | Fields | Status | Notes | |------------------------------------------------------|----------------------|------------------------------------------------------------------------------------------------| -| `spec.listeners[].port` | Not supported* | The configuration is required but ignored. This is due to limitations in the data plane: it cannot dynamically open new ports. Since the Ingress Controller does not manage the data plane deployment, it cannot automatically update the configuration or restart the data plane to apply port changes. | +| `spec.listeners[].port` | Partially supported | Controls `server_port` route-var injection; behaviour is configured via [`listener_port_match_mode`](../reference/configuration-file.md) (`auto` / `explicit` / `off`). The controller cannot dynamically open data plane ports, so APISIX must already listen on the specified port. | | `spec.listeners[].tls.certificateRefs[].group` | Partially supported | Only `""` is supported; other group values cause validation failure. | | `spec.listeners[].tls.certificateRefs[].kind` | Partially supported | Only `Secret` is supported. | | `spec.listeners[].tls.mode` | Partially supported | `Terminate` is implemented; `Passthrough` is effectively unsupported for Gateway listeners. | diff --git a/docs/en/latest/reference/configuration-file.md b/docs/en/latest/reference/configuration-file.md index b01b0294b7..166570ab73 100644 --- a/docs/en/latest/reference/configuration-file.md +++ b/docs/en/latest/reference/configuration-file.md @@ -63,6 +63,12 @@ secure_metrics: false # The secure metrics configuration. exec_adc_timeout: 15s # The timeout for the ADC to execute. # The default value is 15 seconds. +listener_port_match_mode: "auto" # Mode for injecting server_port route vars from Gateway listener ports. + # - "auto": inject when parentRefs explicitly target listeners (sectionName/port) or when multiple listener ports are matched. + # - "explicit": inject only when parentRefs explicitly target listeners. + # - "off": never inject server_port vars. + # The default value is "auto". + provider: type: "apisix" # Provider type. # Value can be "apisix" or "apisix-standalone". diff --git a/docs/en/latest/reference/example.md b/docs/en/latest/reference/example.md index 8c219f8c3b..ef9ba9fa52 100644 --- a/docs/en/latest/reference/example.md +++ b/docs/en/latest/reference/example.md @@ -96,7 +96,7 @@ spec: ❶ The controller name should be customized if you are running multiple distinct instances of the APISIX Ingress Controller in the same cluster (not a single instance with multiple replicas). Each ingress controller instance must use a unique controllerName in its [configuration file](configuration-file.md), and the corresponding GatewayClass should reference that value. -❷ The `port` in the Gateway listener is required but ignored. This is due to limitations in the data plane: it cannot dynamically open new ports. Since the Ingress Controller does not manage the data plane deployment, it cannot automatically update the configuration or restart the data plane to apply port changes. +❷ The `port` in the Gateway listener is used for routing matching based on `listener_port_match_mode` in the controller configuration (`auto`, `explicit`, or `off`). The controller cannot dynamically open new ports on the data plane, so ensure APISIX is configured to listen on the port. ❸ API group of the referenced resource. diff --git a/internal/adc/translator/annotations_test.go b/internal/adc/translator/annotations_test.go index 8c8b1b96ef..7b4d1ec496 100644 --- a/internal/adc/translator/annotations_test.go +++ b/internal/adc/translator/annotations_test.go @@ -21,10 +21,13 @@ import ( "github.com/incubator4/go-resty-expr/expr" "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" adctypes "github.com/apache/apisix-ingress-controller/api/adc" "github.com/apache/apisix-ingress-controller/internal/adc/translator/annotations" "github.com/apache/apisix-ingress-controller/internal/adc/translator/annotations/upstream" + "github.com/apache/apisix-ingress-controller/internal/controller/config" ) type mockParser struct { @@ -342,7 +345,7 @@ func TestTranslateIngressAnnotations(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - translator := &Translator{} + translator := &Translator{ListenerPortMatchMode: config.ListenerPortMatchModeAuto} result := translator.TranslateIngressAnnotations(tt.anno) assert.NotNil(t, result) @@ -350,3 +353,247 @@ func TestTranslateIngressAnnotations(t *testing.T) { }) } } + +func TestAddServerPortVars(t *testing.T) { + tests := []struct { + name string + route *adctypes.Route + ports map[int32]struct{} + expected adctypes.Vars + }{ + { + name: "empty ports map - no vars added", + route: &adctypes.Route{}, + ports: map[int32]struct{}{}, + expected: adctypes.Vars(nil), + }, + { + name: "single port - uses == operator", + route: &adctypes.Route{}, + ports: map[int32]struct{}{ + 9080: {}, + }, + expected: adctypes.Vars{ + { + {StrVal: "server_port"}, + {StrVal: "=="}, + {StrVal: "9080"}, + }, + }, + }, + { + name: "two ports - uses 'in' operator", + route: &adctypes.Route{}, + ports: map[int32]struct{}{ + 9080: {}, + 9081: {}, + }, + expected: adctypes.Vars{ + { + {StrVal: "server_port"}, + {StrVal: "in"}, + {SliceVal: []adctypes.StringOrSlice{ + {StrVal: "9080"}, + {StrVal: "9081"}, + }}, + }, + }, + }, + { + name: "three ports - uses 'in' operator", + route: &adctypes.Route{}, + ports: map[int32]struct{}{ + 80: {}, + 443: {}, + 9080: {}, + }, + expected: adctypes.Vars{ + { + {StrVal: "server_port"}, + {StrVal: "in"}, + {SliceVal: []adctypes.StringOrSlice{ + {StrVal: "80"}, + {StrVal: "443"}, + {StrVal: "9080"}, + }}, + }, + }, + }, + { + name: "vars are appended - preserves existing vars", + route: &adctypes.Route{ + Vars: adctypes.Vars{ + { + {StrVal: "uri"}, + {StrVal: "~~"}, + {StrVal: "^/api"}, + }, + }, + }, + ports: map[int32]struct{}{ + 9080: {}, + }, + expected: adctypes.Vars{ + { + {StrVal: "uri"}, + {StrVal: "~~"}, + {StrVal: "^/api"}, + }, + { + {StrVal: "server_port"}, + {StrVal: "=="}, + {StrVal: "9080"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + addServerPortVars(tt.route, tt.ports) + assert.Equal(t, tt.expected, tt.route.Vars) + }) + } +} + +func TestShouldInjectServerPortVars(t *testing.T) { + sectionName := gatewayv1.SectionName("http-main") + port := gatewayv1.PortNumber(9080) + + tests := []struct { + name string + mode config.ListenerPortMatchMode + parentRefs []gatewayv1.ParentReference + ports map[int32]struct{} + expected bool + }{ + { + name: "empty listener ports", + mode: config.ListenerPortMatchModeAuto, + ports: map[int32]struct{}{}, + expected: false, + }, + { + name: "single port without sectionName", + mode: config.ListenerPortMatchModeAuto, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw"}, + }, + ports: map[int32]struct{}{ + 9080: {}, + }, + expected: false, + }, + { + name: "single port with sectionName", + mode: config.ListenerPortMatchModeAuto, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", SectionName: §ionName}, + }, + ports: map[int32]struct{}{ + 9080: {}, + }, + expected: true, + }, + { + name: "multiple ports without sectionName", + mode: config.ListenerPortMatchModeAuto, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw"}, + }, + ports: map[int32]struct{}{ + 9080: {}, + 9081: {}, + }, + expected: true, + }, + { + name: "explicit mode with multiple ports and no explicit target", + mode: config.ListenerPortMatchModeExplicit, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw"}, + }, + ports: map[int32]struct{}{ + 9080: {}, + 9081: {}, + }, + expected: false, + }, + { + name: "explicit mode with parentRef.port", + mode: config.ListenerPortMatchModeExplicit, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", Port: &port}, + }, + ports: map[int32]struct{}{ + 9080: {}, + }, + expected: true, + }, + { + name: "explicit mode with single port and no explicit target", + mode: config.ListenerPortMatchModeExplicit, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw"}, + }, + ports: map[int32]struct{}{ + 9080: {}, + }, + expected: false, + }, + { + name: "off mode ignores explicit target", + mode: config.ListenerPortMatchModeOff, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", SectionName: §ionName}, + }, + ports: map[int32]struct{}{ + 9080: {}, + 9081: {}, + }, + expected: false, + }, + { + name: "off mode ignores explicit parentRef.port target", + mode: config.ListenerPortMatchModeOff, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", Port: &port}, + }, + ports: map[int32]struct{}{ + 9080: {}, + }, + expected: false, + }, + { + name: "explicit mode: non-Gateway parentRef with port is not treated as explicit target", + mode: config.ListenerPortMatchModeExplicit, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw"}, + {Name: "svc", Kind: ptr.To(gatewayv1.Kind("Service")), Port: &port}, + }, + ports: map[int32]struct{}{ + 9080: {}, + }, + expected: false, + }, + { + name: "auto mode: non-Gateway parentRef with port does not trigger single-port injection", + mode: config.ListenerPortMatchModeAuto, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw"}, + {Name: "svc", Kind: ptr.To(gatewayv1.Kind("Service")), Port: &port}, + }, + ports: map[int32]struct{}{ + 9080: {}, + }, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + translator := &Translator{ListenerPortMatchMode: tt.mode} + assert.Equal(t, tt.expected, translator.shouldInjectServerPortVars(tt.parentRefs, tt.ports)) + }) + } +} diff --git a/internal/adc/translator/grpcroute.go b/internal/adc/translator/grpcroute.go index abe6dfab09..631b34d57e 100644 --- a/internal/adc/translator/grpcroute.go +++ b/internal/adc/translator/grpcroute.go @@ -308,6 +308,19 @@ func (t *Translator) TranslateGRPCRoute(tctx *provider.TranslateContext, grpcRou routes = append(routes, route) } + + // Collect unique listener ports for port-based routing. + listenerPorts := make(map[int32]struct{}) + for _, listener := range tctx.Listeners { + listenerPorts[int32(listener.Port)] = struct{}{} + } + + if t.shouldInjectServerPortVars(tctx.RouteParentRefs, listenerPorts) { + for _, route := range routes { + addServerPortVars(route, listenerPorts) + } + } + service.Routes = routes result.Services = append(result.Services, service) diff --git a/internal/adc/translator/grpcroute_test.go b/internal/adc/translator/grpcroute_test.go new file mode 100644 index 0000000000..df95a35ad6 --- /dev/null +++ b/internal/adc/translator/grpcroute_test.go @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package translator + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + adctypes "github.com/apache/apisix-ingress-controller/api/adc" + "github.com/apache/apisix-ingress-controller/internal/controller/config" + "github.com/apache/apisix-ingress-controller/internal/provider" +) + +func TestTranslateGRPCRouteServerPortVarsByMode(t *testing.T) { + sectionName := gatewayv1.SectionName("grpc-main") + parentPort := gatewayv1.PortNumber(9080) + + singlePortVars := adctypes.Vars{ + { + {StrVal: "server_port"}, + {StrVal: "=="}, + {StrVal: "9080"}, + }, + } + multiPortVars := adctypes.Vars{ + { + {StrVal: "server_port"}, + {StrVal: "in"}, + {SliceVal: []adctypes.StringOrSlice{ + {StrVal: "9080"}, + {StrVal: "9081"}, + }}, + }, + } + + tests := []struct { + name string + mode config.ListenerPortMatchMode + parentRefs []gatewayv1.ParentReference + listeners []gatewayv1.Listener + expected adctypes.Vars + }{ + { + name: "auto mode: no injection for single listener without explicit target", + mode: config.ListenerPortMatchModeAuto, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw"}, + }, + listeners: []gatewayv1.Listener{ + {Name: "grpc-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: nil, + }, + { + name: "auto mode: inject for sectionName target", + mode: config.ListenerPortMatchModeAuto, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", SectionName: §ionName}, + }, + listeners: []gatewayv1.Listener{ + {Name: "grpc-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: singlePortVars, + }, + { + name: "auto mode: inject for port target", + mode: config.ListenerPortMatchModeAuto, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", Port: &parentPort}, + }, + listeners: []gatewayv1.Listener{ + {Name: "grpc-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: singlePortVars, + }, + { + name: "auto mode: inject for multiple listener ports", + mode: config.ListenerPortMatchModeAuto, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw"}, + }, + listeners: []gatewayv1.Listener{ + {Name: "grpc-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9081)}, + {Name: "grpc-alt", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: multiPortVars, + }, + { + name: "auto mode: inject for multiple listener ports when listener names collide across gateways", + mode: config.ListenerPortMatchModeAuto, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw-a"}, + {Name: "gw-b"}, + }, + listeners: []gatewayv1.Listener{ + {Name: "http", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9081)}, + {Name: "http", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: multiPortVars, + }, + { + name: "explicit mode: inject for sectionName target", + mode: config.ListenerPortMatchModeExplicit, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", SectionName: §ionName}, + }, + listeners: []gatewayv1.Listener{ + {Name: "grpc-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: singlePortVars, + }, + { + name: "explicit mode: inject for port target", + mode: config.ListenerPortMatchModeExplicit, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", Port: &parentPort}, + }, + listeners: []gatewayv1.Listener{ + {Name: "grpc-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: singlePortVars, + }, + { + name: "explicit mode: no injection for multiple listener ports without explicit target", + mode: config.ListenerPortMatchModeExplicit, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw"}, + }, + listeners: []gatewayv1.Listener{ + {Name: "grpc-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9081)}, + {Name: "grpc-alt", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: nil, + }, + { + name: "off mode: no injection even with sectionName target", + mode: config.ListenerPortMatchModeOff, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", SectionName: §ionName}, + }, + listeners: []gatewayv1.Listener{ + {Name: "grpc-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: nil, + }, + { + name: "off mode: no injection for multiple listener ports", + mode: config.ListenerPortMatchModeOff, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw"}, + }, + listeners: []gatewayv1.Listener{ + {Name: "grpc-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9081)}, + {Name: "grpc-alt", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: nil, + }, + { + name: "empty mode normalizes to auto", + mode: "", + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", Port: &parentPort}, + }, + listeners: []gatewayv1.Listener{ + {Name: "grpc-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: singlePortVars, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tctx := provider.NewDefaultTranslateContext(context.Background()) + tctx.RouteParentRefs = tt.parentRefs + tctx.Listeners = tt.listeners + + grpcRoute := &gatewayv1.GRPCRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Namespace: "default", + }, + Spec: gatewayv1.GRPCRouteSpec{ + Rules: []gatewayv1.GRPCRouteRule{ + {}, + }, + }, + } + + translator := NewTranslator(logr.Discard(), tt.mode) + got, err := translator.TranslateGRPCRoute(tctx, grpcRoute) + assert.NoError(t, err) + if assert.Len(t, got.Services, 1) && assert.Len(t, got.Services[0].Routes, 1) { + assert.Equal(t, tt.expected, got.Services[0].Routes[0].Vars) + } + }) + } +} diff --git a/internal/adc/translator/httproute.go b/internal/adc/translator/httproute.go index b16192a780..ae9a974b2f 100644 --- a/internal/adc/translator/httproute.go +++ b/internal/adc/translator/httproute.go @@ -20,6 +20,7 @@ package translator import ( "encoding/json" "fmt" + "sort" "strings" "github.com/pkg/errors" @@ -524,6 +525,135 @@ func calculateHTTPRoutePriority(match *gatewayv1.HTTPRouteMatch, ruleIndex int, return priority } +// translateBackendsToUpstreams processes the BackendRefs of an HTTPRouteRule, +// builds upstreams, assigns them to the service (single upstream or traffic-split +// plugin for multiple), and injects fault-injection on backend errors. +func (t *Translator) translateBackendsToUpstreams( + tctx *provider.TranslateContext, + rule gatewayv1.HTTPRouteRule, + httpRoute *gatewayv1.HTTPRoute, + service *adctypes.Service, +) (enableWebsocket *bool, backendErr error) { + upstreams := make([]*adctypes.Upstream, 0) + weightedUpstreams := make([]adctypes.TrafficSplitConfigRuleWeightedUpstream, 0) + + for _, backend := range rule.BackendRefs { + if backend.Namespace == nil { + namespace := gatewayv1.Namespace(httpRoute.Namespace) + backend.Namespace = &namespace + } + upstream := adctypes.NewDefaultUpstream() + upNodes, protocol, err := t.translateBackendRef(tctx, backend.BackendRef, DefaultEndpointFilter) + if err != nil { + backendErr = err + continue + } + if len(upNodes) == 0 { + continue + } + if protocol == internaltypes.AppProtocolWS || protocol == internaltypes.AppProtocolWSS { + enableWebsocket = ptr.To(true) + } + + t.AttachBackendTrafficPolicyToUpstream(backend.BackendRef, tctx.BackendTrafficPolicies, upstream) + upstream.Nodes = upNodes + if upstream.Scheme == "" { + upstream.Scheme = appProtocolToUpstreamScheme(protocol) + } + var ( + kind string + port int32 + ) + if backend.Kind == nil { + kind = internaltypes.KindService + } else { + kind = string(*backend.Kind) + } + if backend.Port != nil { + port = int32(*backend.Port) + } + namespace := string(*backend.Namespace) + name := string(backend.Name) + upstreamName := adctypes.ComposeUpstreamNameForBackendRef(kind, namespace, name, port) + upstream.Name = upstreamName + upstream.ID = id.GenID(upstreamName) + upstreams = append(upstreams, upstream) + } + + // Handle multiple backends with traffic-split plugin + if len(upstreams) == 0 { + // Create a default upstream if no valid backends + service.Upstream = adctypes.NewDefaultUpstream() + } else if len(upstreams) == 1 { + // Single backend - use directly as service upstream + service.Upstream = upstreams[0] + // remove the id and name of the service.upstream, adc schema does not need id and name for it + service.Upstream.ID = "" + service.Upstream.Name = "" + } else { + // Multiple backends - use traffic-split plugin + service.Upstream = upstreams[0] + // remove the id and name of the service.upstream, adc schema does not need id and name for it + service.Upstream.ID = "" + service.Upstream.Name = "" + + upstreams = upstreams[1:] + + if len(upstreams) > 0 { + service.Upstreams = upstreams + } + + // Set weight in traffic-split for the default upstream + weight := apiv2.DefaultWeight + if rule.BackendRefs[0].Weight != nil { + weight = int(*rule.BackendRefs[0].Weight) + } + weightedUpstreams = append(weightedUpstreams, adctypes.TrafficSplitConfigRuleWeightedUpstream{ + Weight: weight, + }) + + // Set other upstreams in traffic-split using upstream_id + for i, upstream := range upstreams { + weight := apiv2.DefaultWeight + // get weight from the backend refs starting from the second backend + if i+1 < len(rule.BackendRefs) && rule.BackendRefs[i+1].Weight != nil { + weight = int(*rule.BackendRefs[i+1].Weight) + } + weightedUpstreams = append(weightedUpstreams, adctypes.TrafficSplitConfigRuleWeightedUpstream{ + UpstreamID: upstream.ID, + Weight: weight, + }) + } + + if len(weightedUpstreams) > 0 { + if service.Plugins == nil { + service.Plugins = make(map[string]any) + } + service.Plugins["traffic-split"] = &adctypes.TrafficSplitConfig{ + Rules: []adctypes.TrafficSplitConfigRule{ + { + WeightedUpstreams: weightedUpstreams, + }, + }, + } + } + } + + if backendErr != nil && (service.Upstream == nil || len(service.Upstream.Nodes) == 0) { + if service.Plugins == nil { + service.Plugins = make(map[string]any) + } + service.Plugins["fault-injection"] = map[string]any{ + "abort": map[string]any{ + "http_status": 500, + "body": "No existing backendRef provided", + }, + } + } + + return enableWebsocket, backendErr +} + func (t *Translator) TranslateHTTPRoute(tctx *provider.TranslateContext, httpRoute *gatewayv1.HTTPRoute) (*TranslateResult, error) { result := &TranslateResult{} @@ -544,127 +674,7 @@ func (t *Translator) TranslateHTTPRoute(tctx *provider.TranslateContext, httpRou service.ID = id.GenID(service.Name) service.Hosts = hosts - var ( - upstreams = make([]*adctypes.Upstream, 0) - weightedUpstreams = make([]adctypes.TrafficSplitConfigRuleWeightedUpstream, 0) - backendErr error - enableWebsocket *bool - ) - - for _, backend := range rule.BackendRefs { - if backend.Namespace == nil { - namespace := gatewayv1.Namespace(httpRoute.Namespace) - backend.Namespace = &namespace - } - upstream := adctypes.NewDefaultUpstream() - upNodes, protocol, err := t.translateBackendRef(tctx, backend.BackendRef, DefaultEndpointFilter) - if err != nil { - backendErr = err - continue - } - if len(upNodes) == 0 { - continue - } - if protocol == internaltypes.AppProtocolWS || protocol == internaltypes.AppProtocolWSS { - enableWebsocket = ptr.To(true) - } - - t.AttachBackendTrafficPolicyToUpstream(backend.BackendRef, tctx.BackendTrafficPolicies, upstream) - upstream.Nodes = upNodes - if upstream.Scheme == "" { - upstream.Scheme = appProtocolToUpstreamScheme(protocol) - } - var ( - kind string - port int32 - ) - if backend.Kind == nil { - kind = internaltypes.KindService - } else { - kind = string(*backend.Kind) - } - if backend.Port != nil { - port = int32(*backend.Port) - } - namespace := string(*backend.Namespace) - name := string(backend.Name) - upstreamName := adctypes.ComposeUpstreamNameForBackendRef(kind, namespace, name, port) - upstream.Name = upstreamName - upstream.ID = id.GenID(upstreamName) - upstreams = append(upstreams, upstream) - } - - // Handle multiple backends with traffic-split plugin - if len(upstreams) == 0 { - // Create a default upstream if no valid backends - upstream := adctypes.NewDefaultUpstream() - service.Upstream = upstream - } else if len(upstreams) == 1 { - // Single backend - use directly as service upstream - service.Upstream = upstreams[0] - // remove the id and name of the service.upstream, adc schema does not need id and name for it - service.Upstream.ID = "" - service.Upstream.Name = "" - } else { - // Multiple backends - use traffic-split plugin - service.Upstream = upstreams[0] - // remove the id and name of the service.upstream, adc schema does not need id and name for it - service.Upstream.ID = "" - service.Upstream.Name = "" - - upstreams = upstreams[1:] - - if len(upstreams) > 0 { - service.Upstreams = upstreams - } - - // Set weight in traffic-split for the default upstream - weight := apiv2.DefaultWeight - if rule.BackendRefs[0].Weight != nil { - weight = int(*rule.BackendRefs[0].Weight) - } - weightedUpstreams = append(weightedUpstreams, adctypes.TrafficSplitConfigRuleWeightedUpstream{ - Weight: weight, - }) - - // Set other upstreams in traffic-split using upstream_id - for i, upstream := range upstreams { - weight := apiv2.DefaultWeight - // get weight from the backend refs starting from the second backend - if i+1 < len(rule.BackendRefs) && rule.BackendRefs[i+1].Weight != nil { - weight = int(*rule.BackendRefs[i+1].Weight) - } - weightedUpstreams = append(weightedUpstreams, adctypes.TrafficSplitConfigRuleWeightedUpstream{ - UpstreamID: upstream.ID, - Weight: weight, - }) - } - - if len(weightedUpstreams) > 0 { - if service.Plugins == nil { - service.Plugins = make(map[string]any) - } - service.Plugins["traffic-split"] = &adctypes.TrafficSplitConfig{ - Rules: []adctypes.TrafficSplitConfigRule{ - { - WeightedUpstreams: weightedUpstreams, - }, - }, - } - } - } - - if backendErr != nil && (service.Upstream == nil || len(service.Upstream.Nodes) == 0) { - if service.Plugins == nil { - service.Plugins = make(map[string]any) - } - service.Plugins["fault-injection"] = map[string]any{ - "abort": map[string]any{ - "http_status": 500, - "body": "No existing backendRef provided", - }, - } - } + enableWebsocket, _ := t.translateBackendsToUpstreams(tctx, rule, httpRoute, service) t.fillPluginsFromHTTPRouteFilters(service.Plugins, httpRoute.GetNamespace(), rule.Filters, rule.Matches, tctx) @@ -701,6 +711,21 @@ func (t *Translator) TranslateHTTPRoute(tctx *provider.TranslateContext, httpRou routes = append(routes, route) } + + // Collect unique listener ports for port-based routing + listenerPorts := make(map[int32]struct{}) + for _, listener := range tctx.Listeners { + listenerPorts[int32(listener.Port)] = struct{}{} + } + + // Add server_port matching only when a route explicitly targets a listener + // or when multiple listener ports need to be disambiguated. + if t.shouldInjectServerPortVars(tctx.RouteParentRefs, listenerPorts) { + for _, route := range routes { + addServerPortVars(route, listenerPorts) + } + } + t.fillHTTPRoutePoliciesForHTTPRoute(tctx, routes, rule) service.Routes = routes @@ -850,3 +875,41 @@ func appProtocolToUpstreamScheme(appProtocol string) string { return "" } } + +func addServerPortVars(route *adctypes.Route, ports map[int32]struct{}) { + if len(ports) == 0 { + return + } + + // For single port, use exact match + if len(ports) == 1 { + for port := range ports { + portVar := []adctypes.StringOrSlice{ + {StrVal: "server_port"}, + {StrVal: "=="}, + {StrVal: fmt.Sprintf("%d", port)}, + } + route.Vars = append(route.Vars, portVar) + return + } + } + + // For multiple ports, use "in" operator + // Sort ports for deterministic output + sortedPorts := make([]int, 0, len(ports)) + for port := range ports { + sortedPorts = append(sortedPorts, int(port)) + } + sort.Ints(sortedPorts) + + portList := make([]adctypes.StringOrSlice, 0, len(ports)) + for _, port := range sortedPorts { + portList = append(portList, adctypes.StringOrSlice{StrVal: fmt.Sprintf("%d", port)}) + } + portVar := []adctypes.StringOrSlice{ + {StrVal: "server_port"}, + {StrVal: "in"}, + {SliceVal: portList}, + } + route.Vars = append(route.Vars, portVar) +} diff --git a/internal/adc/translator/httproute_test.go b/internal/adc/translator/httproute_test.go new file mode 100644 index 0000000000..6905e96ed7 --- /dev/null +++ b/internal/adc/translator/httproute_test.go @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package translator + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + adctypes "github.com/apache/apisix-ingress-controller/api/adc" + "github.com/apache/apisix-ingress-controller/internal/controller/config" + "github.com/apache/apisix-ingress-controller/internal/provider" +) + +func TestTranslateHTTPRouteServerPortVarsByMode(t *testing.T) { + sectionName := gatewayv1.SectionName("http-main") + parentPort := gatewayv1.PortNumber(9080) + pathMatchType := gatewayv1.PathMatchPathPrefix + pathValue := "/" + + singlePortVars := adctypes.Vars{ + { + {StrVal: "server_port"}, + {StrVal: "=="}, + {StrVal: "9080"}, + }, + } + multiPortVars := adctypes.Vars{ + { + {StrVal: "server_port"}, + {StrVal: "in"}, + {SliceVal: []adctypes.StringOrSlice{ + {StrVal: "9080"}, + {StrVal: "9081"}, + }}, + }, + } + + tests := []struct { + name string + mode config.ListenerPortMatchMode + parentRefs []gatewayv1.ParentReference + listeners []gatewayv1.Listener + expected adctypes.Vars + }{ + { + name: "auto mode: no injection for single listener without explicit target", + mode: config.ListenerPortMatchModeAuto, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw"}, + }, + listeners: []gatewayv1.Listener{ + {Name: "http-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: nil, + }, + { + name: "auto mode: inject for sectionName target", + mode: config.ListenerPortMatchModeAuto, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", SectionName: §ionName}, + }, + listeners: []gatewayv1.Listener{ + {Name: "http-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: singlePortVars, + }, + { + name: "auto mode: inject for port target", + mode: config.ListenerPortMatchModeAuto, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", Port: &parentPort}, + }, + listeners: []gatewayv1.Listener{ + {Name: "http-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: singlePortVars, + }, + { + name: "auto mode: inject for multiple listener ports", + mode: config.ListenerPortMatchModeAuto, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw"}, + }, + listeners: []gatewayv1.Listener{ + {Name: "http-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9081)}, + {Name: "http-alt", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: multiPortVars, + }, + { + name: "auto mode: inject for multiple listener ports when listener names collide across gateways", + mode: config.ListenerPortMatchModeAuto, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw-a"}, + {Name: "gw-b"}, + }, + listeners: []gatewayv1.Listener{ + {Name: "http", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9081)}, + {Name: "http", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: multiPortVars, + }, + { + name: "explicit mode: inject for sectionName target", + mode: config.ListenerPortMatchModeExplicit, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", SectionName: §ionName}, + }, + listeners: []gatewayv1.Listener{ + {Name: "http-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: singlePortVars, + }, + { + name: "explicit mode: inject for port target", + mode: config.ListenerPortMatchModeExplicit, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", Port: &parentPort}, + }, + listeners: []gatewayv1.Listener{ + {Name: "http-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: singlePortVars, + }, + { + name: "explicit mode: no injection for multiple listener ports without explicit target", + mode: config.ListenerPortMatchModeExplicit, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw"}, + }, + listeners: []gatewayv1.Listener{ + {Name: "http-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9081)}, + {Name: "http-alt", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: nil, + }, + { + name: "off mode: no injection even with sectionName target", + mode: config.ListenerPortMatchModeOff, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", SectionName: §ionName}, + }, + listeners: []gatewayv1.Listener{ + {Name: "http-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: nil, + }, + { + name: "off mode: no injection for multiple listener ports", + mode: config.ListenerPortMatchModeOff, + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw"}, + }, + listeners: []gatewayv1.Listener{ + {Name: "http-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9081)}, + {Name: "http-alt", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: nil, + }, + { + name: "empty mode normalizes to auto", + mode: "", + parentRefs: []gatewayv1.ParentReference{ + {Name: "gw", Port: &parentPort}, + }, + listeners: []gatewayv1.Listener{ + {Name: "http-main", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(9080)}, + }, + expected: singlePortVars, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tctx := provider.NewDefaultTranslateContext(context.Background()) + tctx.RouteParentRefs = tt.parentRefs + tctx.Listeners = tt.listeners + + httpRoute := &gatewayv1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Namespace: "default", + }, + Spec: gatewayv1.HTTPRouteSpec{ + Rules: []gatewayv1.HTTPRouteRule{ + { + Matches: []gatewayv1.HTTPRouteMatch{ + { + Path: &gatewayv1.HTTPPathMatch{ + Type: &pathMatchType, + Value: &pathValue, + }, + }, + }, + }, + }, + }, + } + + translator := NewTranslator(logr.Discard(), tt.mode) + got, err := translator.TranslateHTTPRoute(tctx, httpRoute) + assert.NoError(t, err) + if assert.Len(t, got.Services, 1) && assert.Len(t, got.Services[0].Routes, 1) { + assert.Equal(t, tt.expected, got.Services[0].Routes[0].Vars) + } + }) + } +} diff --git a/internal/adc/translator/translator.go b/internal/adc/translator/translator.go index aeaef2509b..e294c7dad4 100644 --- a/internal/adc/translator/translator.go +++ b/internal/adc/translator/translator.go @@ -19,17 +19,72 @@ package translator import ( "github.com/go-logr/logr" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" adctypes "github.com/apache/apisix-ingress-controller/api/adc" + "github.com/apache/apisix-ingress-controller/internal/controller/config" ) type Translator struct { - Log logr.Logger + Log logr.Logger + ListenerPortMatchMode config.ListenerPortMatchMode } -func NewTranslator(log logr.Logger) *Translator { +func normalizeMode(mode config.ListenerPortMatchMode) config.ListenerPortMatchMode { + switch mode { + case "", config.ListenerPortMatchModeAuto: + return config.ListenerPortMatchModeAuto + case config.ListenerPortMatchModeExplicit, config.ListenerPortMatchModeOff: + return mode + default: + return config.ListenerPortMatchModeAuto + } +} + +func NewTranslator(log logr.Logger, mode config.ListenerPortMatchMode) *Translator { return &Translator{ - Log: log.WithName("translator"), + Log: log.WithName("translator"), + ListenerPortMatchMode: normalizeMode(mode), + } +} + +func hasExplicitListenerTarget(parentRefs []gatewayv1.ParentReference) bool { + for _, parentRef := range parentRefs { + // Skip non-Gateway parentRefs (e.g. GAMMA Service mesh refs) — they + // are not relevant to listener port injection. + if parentRef.Kind != nil && *parentRef.Kind != "Gateway" { + continue + } + if parentRef.SectionName != nil && *parentRef.SectionName != "" { + return true + } + if parentRef.Port != nil { + return true + } + } + + return false +} + +func (t *Translator) shouldInjectServerPortVars(parentRefs []gatewayv1.ParentReference, ports map[int32]struct{}) bool { + if len(ports) == 0 { + return false + } + + explicit := hasExplicitListenerTarget(parentRefs) + + switch t.ListenerPortMatchMode { + case config.ListenerPortMatchModeOff: + if explicit { + t.Log.V(1).Info("listener_port_match_mode is 'off'; ignoring explicit listener targeting", "parent_refs", len(parentRefs)) + } + return false + case config.ListenerPortMatchModeExplicit: + return explicit + case config.ListenerPortMatchModeAuto: + return explicit || len(ports) > 1 + default: + return explicit || len(ports) > 1 } } diff --git a/internal/controller/config/config.go b/internal/controller/config/config.go index 31ec1783f8..2f885e0216 100644 --- a/internal/controller/config/config.go +++ b/internal/controller/config/config.go @@ -56,7 +56,8 @@ func NewDefaultConfig() *Config { SyncPeriod: types.TimeDuration{Duration: 1 * time.Hour}, InitSyncDelay: types.TimeDuration{Duration: 20 * time.Minute}, }, - Webhook: NewWebhookConfig(), + Webhook: NewWebhookConfig(), + ListenerPortMatchMode: ListenerPortMatchModeAuto, } } @@ -122,6 +123,15 @@ func (c *Config) Validate() error { if c.ControllerName == "" { return fmt.Errorf("controller_name is required") } + + if c.ListenerPortMatchMode != "" { + switch c.ListenerPortMatchMode { + case ListenerPortMatchModeAuto, ListenerPortMatchModeExplicit, ListenerPortMatchModeOff: + default: + return fmt.Errorf("invalid listener_port_match_mode: %q (must be auto, explicit, or off)", c.ListenerPortMatchMode) + } + } + if err := validateProvider(c.ProviderConfig); err != nil { return err } diff --git a/internal/controller/config/config_test.go b/internal/controller/config/config_test.go new file mode 100644 index 0000000000..97224dbf11 --- /dev/null +++ b/internal/controller/config/config_test.go @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewDefaultConfigListenerPortMatchMode(t *testing.T) { + cfg := NewDefaultConfig() + assert.Equal(t, ListenerPortMatchModeAuto, cfg.ListenerPortMatchMode) +} + +func TestConfigValidateListenerPortMatchMode(t *testing.T) { + tests := []struct { + name string + mode ListenerPortMatchMode + expectErr bool + }{ + { + name: "default auto", + mode: ListenerPortMatchModeAuto, + expectErr: false, + }, + { + name: "explicit", + mode: ListenerPortMatchModeExplicit, + expectErr: false, + }, + { + name: "off", + mode: ListenerPortMatchModeOff, + expectErr: false, + }, + { + name: "empty mode is allowed", + mode: "", + expectErr: false, + }, + { + name: "invalid mode", + mode: "invalid", + expectErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := NewDefaultConfig() + cfg.ListenerPortMatchMode = tt.mode + + err := cfg.Validate() + if tt.expectErr { + assert.Error(t, err) + assert.ErrorContains(t, err, "invalid listener_port_match_mode") + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/internal/controller/config/types.go b/internal/controller/config/types.go index e00ef1f493..7e470d1c12 100644 --- a/internal/controller/config/types.go +++ b/internal/controller/config/types.go @@ -28,6 +28,14 @@ const ( ProviderTypeAPISIX ProviderType = "apisix" ) +type ListenerPortMatchMode string + +const ( + ListenerPortMatchModeAuto ListenerPortMatchMode = "auto" + ListenerPortMatchModeExplicit ListenerPortMatchMode = "explicit" + ListenerPortMatchModeOff ListenerPortMatchMode = "off" +) + const ( // IngressAPISIXLeader is the default election id for the controller // leader election. @@ -55,20 +63,21 @@ const ( // Config contains all config items which are necessary for // apisix-ingress-controller's running. type Config struct { - LogLevel string `json:"log_level" yaml:"log_level"` - ControllerName string `json:"controller_name" yaml:"controller_name"` - LeaderElectionID string `json:"leader_election_id" yaml:"leader_election_id"` - MetricsAddr string `json:"metrics_addr" yaml:"metrics_addr"` - ServerAddr string `json:"server_addr" yaml:"server_addr"` - EnableServer bool `json:"enable_server" yaml:"enable_server"` - EnableHTTP2 bool `json:"enable_http2" yaml:"enable_http2"` - ProbeAddr string `json:"probe_addr" yaml:"probe_addr"` - SecureMetrics bool `json:"secure_metrics" yaml:"secure_metrics"` - LeaderElection *LeaderElection `json:"leader_election" yaml:"leader_election"` - ExecADCTimeout types.TimeDuration `json:"exec_adc_timeout" yaml:"exec_adc_timeout"` - ProviderConfig ProviderConfig `json:"provider" yaml:"provider"` - Webhook *WebhookConfig `json:"webhook" yaml:"webhook"` - DisableGatewayAPI bool `json:"disable_gateway_api" yaml:"disable_gateway_api"` + LogLevel string `json:"log_level" yaml:"log_level"` + ControllerName string `json:"controller_name" yaml:"controller_name"` + LeaderElectionID string `json:"leader_election_id" yaml:"leader_election_id"` + MetricsAddr string `json:"metrics_addr" yaml:"metrics_addr"` + ServerAddr string `json:"server_addr" yaml:"server_addr"` + EnableServer bool `json:"enable_server" yaml:"enable_server"` + EnableHTTP2 bool `json:"enable_http2" yaml:"enable_http2"` + ProbeAddr string `json:"probe_addr" yaml:"probe_addr"` + SecureMetrics bool `json:"secure_metrics" yaml:"secure_metrics"` + LeaderElection *LeaderElection `json:"leader_election" yaml:"leader_election"` + ExecADCTimeout types.TimeDuration `json:"exec_adc_timeout" yaml:"exec_adc_timeout"` + ProviderConfig ProviderConfig `json:"provider" yaml:"provider"` + Webhook *WebhookConfig `json:"webhook" yaml:"webhook"` + DisableGatewayAPI bool `json:"disable_gateway_api" yaml:"disable_gateway_api"` + ListenerPortMatchMode ListenerPortMatchMode `json:"listener_port_match_mode" yaml:"listener_port_match_mode"` } type GatewayConfig struct { diff --git a/internal/controller/context.go b/internal/controller/context.go index 5398f0448f..686dd046c7 100644 --- a/internal/controller/context.go +++ b/internal/controller/context.go @@ -27,6 +27,7 @@ type RouteParentRefContext struct { ListenerName string Listener *gatewayv1.Listener + Listeners []gatewayv1.Listener Conditions []metav1.Condition } diff --git a/internal/controller/grpcroute_controller.go b/internal/controller/grpcroute_controller.go index 3b4234171a..e15e09865b 100644 --- a/internal/controller/grpcroute_controller.go +++ b/internal/controller/grpcroute_controller.go @@ -200,8 +200,13 @@ func (r *GRPCRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( acceptStatus.status = false acceptStatus.msg = err.Error() } - if gateway.Listener != nil { - tctx.Listeners = append(tctx.Listeners, *gateway.Listener) + // Populate listeners for port-based routing + // Use Listeners slice if available (multiple listener support) + if len(gateway.Listeners) > 0 { + tctx.Listeners = appendListeners(tctx.Listeners, gateway.Listeners...) + } else if gateway.Listener != nil { + // Fallback for backward compatibility + tctx.Listeners = appendListeners(tctx.Listeners, *gateway.Listener) } } diff --git a/internal/controller/httproute_controller.go b/internal/controller/httproute_controller.go index 34615b9f55..c183587e49 100644 --- a/internal/controller/httproute_controller.go +++ b/internal/controller/httproute_controller.go @@ -183,6 +183,14 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( acceptStatus.status = false acceptStatus.msg = err.Error() } + // Populate listeners for port-based routing + // Use Listeners slice if available (multiple listener support) + if len(gateway.Listeners) > 0 { + tctx.Listeners = appendListeners(tctx.Listeners, gateway.Listeners...) + } else if gateway.Listener != nil { + // Fallback for backward compatibility + tctx.Listeners = appendListeners(tctx.Listeners, *gateway.Listener) + } } var backendRefErr error diff --git a/internal/controller/utils.go b/internal/controller/utils.go index a00ef9165b..eb981beae6 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -362,6 +362,10 @@ func ParseRouteParentRefs( reason := gatewayv1.RouteReasonNoMatchingParent var listenerName string var matchedListener gatewayv1.Listener + var matchedListeners []gatewayv1.Listener + + // Track if sectionName was explicitly specified + sectionNameSpecified := parentRef.SectionName != nil && *parentRef.SectionName != "" for _, listener := range gateway.Spec.Listeners { if parentRef.SectionName != nil { @@ -385,7 +389,6 @@ func ParseRouteParentRefs( continue } - listenerName = string(listener.Name) ok, err := routeMatchesListenerAllowedRoutes(ctx, mgrc, route, listener.AllowedRoutes, gateway.Namespace, parentRef.Namespace) if err != nil { log.Error(err, "failed matching listener to a route for gateway", @@ -400,9 +403,23 @@ func ParseRouteParentRefs( // TODO: check if the listener status is programmed + if sectionNameSpecified { + listenerName = string(listener.Name) + } + + if !matched { + // First match - store for backward compatibility + matchedListener = listener + } + + // Always add to the list of matched listeners + matchedListeners = append(matchedListeners, listener) matched = true - matchedListener = listener - break + + // Only break if sectionName was explicitly specified + if sectionNameSpecified { + break + } } if matched { @@ -410,6 +427,7 @@ func ParseRouteParentRefs( Gateway: &gateway, ListenerName: listenerName, Listener: &matchedListener, + Listeners: matchedListeners, Conditions: []metav1.Condition{{ Type: string(gatewayv1.RouteConditionAccepted), Status: metav1.ConditionTrue, @@ -421,7 +439,8 @@ func ParseRouteParentRefs( gateways = append(gateways, RouteParentRefContext{ Gateway: &gateway, ListenerName: listenerName, - Listener: &matchedListener, + Listener: nil, + Listeners: matchedListeners, Conditions: []metav1.Condition{{ Type: string(gatewayv1.RouteConditionAccepted), Status: metav1.ConditionFalse, @@ -1103,29 +1122,15 @@ func getUnionOfGatewayHostnames(gateways []RouteParentRefContext) ([]gatewayv1.H hostnames := make([]gatewayv1.Hostname, 0) for _, gateway := range gateways { - if gateway.ListenerName != "" { - // If a listener name is specified, only check that listener - for _, listener := range gateway.Gateway.Spec.Listeners { - if string(listener.Name) == gateway.ListenerName { - // If a listener does not specify a hostname, it can match any hostname - if listener.Hostname == nil { - return nil, true - } - hostnames = append(hostnames, *listener.Hostname) - break - } + for _, listener := range listenersForGatewayContext(gateway) { + // Only consider listeners that can effectively configure hostnames (HTTP, HTTPS, or TLS). + if !isListenerHostnameEffective(listener) { + continue } - } else { - // Otherwise, check all listeners - for _, listener := range gateway.Gateway.Spec.Listeners { - // Only consider listeners that can effectively configure hostnames (HTTP, HTTPS, or TLS) - if isListenerHostnameEffective(listener) { - if listener.Hostname == nil { - return nil, true - } - hostnames = append(hostnames, *listener.Hostname) - } + if listener.Hostname == nil { + return nil, true } + hostnames = append(hostnames, *listener.Hostname) } } @@ -1140,19 +1145,15 @@ func getUnionOfGatewayHostnames(gateways []RouteParentRefContext) ([]gatewayv1.H // - If none of the above, return an empty string func getMinimumHostnameIntersection(gateways []RouteParentRefContext, hostname gatewayv1.Hostname) gatewayv1.Hostname { for _, gateway := range gateways { - for _, listener := range gateway.Gateway.Spec.Listeners { - // If a listener name is specified, only check that listener - // If the listener name is not specified, check all listeners - if gateway.ListenerName == "" || gateway.ListenerName == string(listener.Name) { - if listener.Hostname == nil || *listener.Hostname == "" { - return hostname - } - if HostnamesMatch(string(*listener.Hostname), string(hostname)) { - return hostname - } - if HostnamesMatch(string(hostname), string(*listener.Hostname)) { - return *listener.Hostname - } + for _, listener := range listenersForGatewayContext(gateway) { + if listener.Hostname == nil || *listener.Hostname == "" { + return hostname + } + if HostnamesMatch(string(*listener.Hostname), string(hostname)) { + return hostname + } + if HostnamesMatch(string(hostname), string(*listener.Hostname)) { + return *listener.Hostname } } } @@ -1160,6 +1161,16 @@ func getMinimumHostnameIntersection(gateways []RouteParentRefContext, hostname g return "" } +func listenersForGatewayContext(gw RouteParentRefContext) []gatewayv1.Listener { + if len(gw.Listeners) > 0 { + return gw.Listeners + } + if gw.Listener != nil { + return []gatewayv1.Listener{*gw.Listener} + } + return nil +} + // isListenerHostnameEffective checks if a listener can specify a hostname to match the hostname in the request // Basically, check if the listener uses HTTP, HTTPS, or TLS protocol func isListenerHostnameEffective(listener gatewayv1.Listener) bool { @@ -1168,6 +1179,13 @@ func isListenerHostnameEffective(listener gatewayv1.Listener) bool { listener.Protocol == gatewayv1.TLSProtocolType } +// appendListeners appends listeners without de-duplication. +// Route translation aggregates listeners across multiple Gateways, and listener +// names are only unique within a single Gateway. +func appendListeners(target []gatewayv1.Listener, source ...gatewayv1.Listener) []gatewayv1.Listener { + return append(target, source...) +} + func isRouteAccepted(gateways []RouteParentRefContext) bool { for _, gateway := range gateways { for _, condition := range gateway.Conditions { diff --git a/internal/controller/utils_hostname_test.go b/internal/controller/utils_hostname_test.go new file mode 100644 index 0000000000..031b0fe5ab --- /dev/null +++ b/internal/controller/utils_hostname_test.go @@ -0,0 +1,251 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +func TestListenersForGatewayContext(t *testing.T) { + hostname := gatewayv1.Hostname("example.com") + listener := gatewayv1.Listener{ + Name: "http", + Protocol: gatewayv1.HTTPProtocolType, + Port: gatewayv1.PortNumber(80), + Hostname: &hostname, + } + + tests := []struct { + name string + context RouteParentRefContext + expected []gatewayv1.Listener + }{ + { + name: "prefer listeners slice when present", + context: RouteParentRefContext{ + Listeners: []gatewayv1.Listener{ + listener, + { + Name: "https", + Protocol: gatewayv1.HTTPSProtocolType, + Port: gatewayv1.PortNumber(443), + }, + }, + Listener: &gatewayv1.Listener{ + Name: "ignored", + }, + }, + expected: []gatewayv1.Listener{ + listener, + { + Name: "https", + Protocol: gatewayv1.HTTPSProtocolType, + Port: gatewayv1.PortNumber(443), + }, + }, + }, + { + name: "fallback to single listener pointer", + context: RouteParentRefContext{ + Listener: &listener, + }, + expected: []gatewayv1.Listener{ + listener, + }, + }, + { + name: "no matched listeners", + context: RouteParentRefContext{}, + expected: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, listenersForGatewayContext(tt.context)) + }) + } +} + +func TestGetUnionOfGatewayHostnames(t *testing.T) { + fooHostname := gatewayv1.Hostname("foo.example.com") + barHostname := gatewayv1.Hostname("bar.example.com") + + t.Run("uses all matched listeners and ignores non-hostname-effective listeners", func(t *testing.T) { + gateways := []RouteParentRefContext{ + { + Listeners: []gatewayv1.Listener{ + {Name: "foo", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(80), Hostname: &fooHostname}, + {Name: "bar", Protocol: gatewayv1.HTTPSProtocolType, Port: gatewayv1.PortNumber(443), Hostname: &barHostname}, + {Name: "tcp", Protocol: gatewayv1.TCPProtocolType, Port: gatewayv1.PortNumber(9100)}, + }, + }, + } + + hostnames, matchAny := getUnionOfGatewayHostnames(gateways) + assert.False(t, matchAny) + assert.Equal(t, []gatewayv1.Hostname{fooHostname, barHostname}, hostnames) + }) + + t.Run("listener without hostname matches any hostname", func(t *testing.T) { + gateways := []RouteParentRefContext{ + { + Listeners: []gatewayv1.Listener{ + {Name: "http", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(80)}, + }, + }, + } + + hostnames, matchAny := getUnionOfGatewayHostnames(gateways) + assert.True(t, matchAny) + assert.Nil(t, hostnames) + }) +} + +func TestGetMinimumHostnameIntersection(t *testing.T) { + fooHostname := gatewayv1.Hostname("foo.example.com") + routeHostname := gatewayv1.Hostname("foo.example.com") + wildcardHostname := gatewayv1.Hostname("*.example.com") + + t.Run("matches across multiple listeners without sectionName", func(t *testing.T) { + gateways := []RouteParentRefContext{ + { + Listeners: []gatewayv1.Listener{ + {Name: "foo", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(80), Hostname: &fooHostname}, + {Name: "wildcard", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(80), Hostname: &wildcardHostname}, + }, + }, + } + + assert.Equal(t, routeHostname, getMinimumHostnameIntersection(gateways, routeHostname)) + }) + + t.Run("returns empty when there is no listener match", func(t *testing.T) { + gateways := []RouteParentRefContext{ + { + Listeners: []gatewayv1.Listener{ + {Name: "foo", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(80), Hostname: &fooHostname}, + }, + }, + } + unmatched := gatewayv1.Hostname("bar.example.com") + assert.Equal(t, gatewayv1.Hostname(""), getMinimumHostnameIntersection(gateways, unmatched)) + }) +} + +func TestFilterHostnamesWithMatchedListeners(t *testing.T) { + fooHostname := gatewayv1.Hostname("foo.example.com") + barHostname := gatewayv1.Hostname("bar.example.com") + + route := &gatewayv1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Spec: gatewayv1.HTTPRouteSpec{ + Hostnames: []gatewayv1.Hostname{ + fooHostname, + barHostname, + }, + }, + } + + gateways := []RouteParentRefContext{ + { + Listeners: []gatewayv1.Listener{ + {Name: "foo", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(80), Hostname: &fooHostname}, + {Name: "bar", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(80), Hostname: &barHostname}, + }, + }, + } + + filtered, err := filterHostnames(gateways, route.DeepCopy()) + assert.NoError(t, err) + assert.Equal(t, []gatewayv1.Hostname{fooHostname, barHostname}, filtered.Spec.Hostnames) +} + +func TestFilterHostnamesNoMatchedListeners(t *testing.T) { + fooHostname := gatewayv1.Hostname("foo.example.com") + barHostname := gatewayv1.Hostname("bar.example.com") + + route := &gatewayv1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + Spec: gatewayv1.HTTPRouteSpec{ + Hostnames: []gatewayv1.Hostname{ + barHostname, + }, + }, + } + + gateways := []RouteParentRefContext{ + { + Listeners: []gatewayv1.Listener{ + {Name: "foo", Protocol: gatewayv1.HTTPProtocolType, Port: gatewayv1.PortNumber(80), Hostname: &fooHostname}, + }, + }, + } + + _, err := filterHostnames(gateways, route.DeepCopy()) + assert.ErrorIs(t, err, ErrNoMatchingListenerHostname) +} + +func TestAppendListeners(t *testing.T) { + listenerA := gatewayv1.Listener{Name: "a", Port: 80} + listenerB := gatewayv1.Listener{Name: "b", Port: 81} + listenerA2 := gatewayv1.Listener{Name: "a", Port: 82} + listenerA3 := gatewayv1.Listener{Name: "a", Port: 80} + + tests := []struct { + name string + target []gatewayv1.Listener + source []gatewayv1.Listener + expected []gatewayv1.Listener + }{ + { + name: "empty target, add listeners", + target: nil, + source: []gatewayv1.Listener{listenerA, listenerB}, + expected: []gatewayv1.Listener{listenerA, listenerB}, + }, + { + name: "preserves same listener names from different gateways", + target: []gatewayv1.Listener{listenerA}, + source: []gatewayv1.Listener{listenerA, listenerB}, + expected: []gatewayv1.Listener{listenerA, listenerA, listenerB}, + }, + { + name: "preserves all listeners when names collide", + target: []gatewayv1.Listener{listenerA}, + source: []gatewayv1.Listener{listenerB, listenerA2, listenerA3}, + expected: []gatewayv1.Listener{listenerA, listenerB, listenerA2, listenerA3}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, appendListeners(tt.target, tt.source...)) + }) + } +} diff --git a/internal/manager/run.go b/internal/manager/run.go index 78bc30e506..315644dac2 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -190,9 +190,10 @@ func Run(ctx context.Context, logger logr.Logger) error { providerType := string(config.ControllerConfig.ProviderConfig.Type) providerOptions := &provider.Options{ - SyncTimeout: config.ControllerConfig.ExecADCTimeout.Duration, - SyncPeriod: config.ControllerConfig.ProviderConfig.SyncPeriod.Duration, - InitSyncDelay: config.ControllerConfig.ProviderConfig.InitSyncDelay.Duration, + SyncTimeout: config.ControllerConfig.ExecADCTimeout.Duration, + SyncPeriod: config.ControllerConfig.ProviderConfig.SyncPeriod.Duration, + InitSyncDelay: config.ControllerConfig.ProviderConfig.InitSyncDelay.Duration, + ListenerPortMatchMode: config.ControllerConfig.ListenerPortMatchMode, } provider, err := provider.New(providerType, logger, updater.Writer(), readier, providerOptions) if err != nil { diff --git a/internal/provider/apisix/provider.go b/internal/provider/apisix/provider.go index 029675e219..ef6e3fc835 100644 --- a/internal/provider/apisix/provider.go +++ b/internal/provider/apisix/provider.go @@ -86,7 +86,7 @@ func New(log logr.Logger, updater status.Updater, readier readiness.ReadinessMan return &apisixProvider{ client: cli, Options: o, - translator: translator.NewTranslator(log), + translator: translator.NewTranslator(log, o.ListenerPortMatchMode), updater: updater, readier: readier, syncCh: make(chan struct{}, 1), diff --git a/internal/provider/options.go b/internal/provider/options.go index dbb0760bf0..c47e7ce913 100644 --- a/internal/provider/options.go +++ b/internal/provider/options.go @@ -19,6 +19,8 @@ package provider import ( "time" + + "github.com/apache/apisix-ingress-controller/internal/controller/config" ) type Option interface { @@ -31,6 +33,7 @@ type Options struct { InitSyncDelay time.Duration DefaultBackendMode string DefaultResolveEndpoints bool + ListenerPortMatchMode config.ListenerPortMatchMode } func (o *Options) ApplyToList(lo *Options) { @@ -49,6 +52,9 @@ func (o *Options) ApplyToList(lo *Options) { if o.DefaultResolveEndpoints { lo.DefaultResolveEndpoints = o.DefaultResolveEndpoints } + if o.ListenerPortMatchMode != "" { + lo.ListenerPortMatchMode = o.ListenerPortMatchMode + } } func (o *Options) ApplyOptions(opts []Option) *Options { diff --git a/test/e2e/framework/manifests/apisix.yaml b/test/e2e/framework/manifests/apisix.yaml index 310398e157..90f8845a5e 100644 --- a/test/e2e/framework/manifests/apisix.yaml +++ b/test/e2e/framework/manifests/apisix.yaml @@ -46,7 +46,10 @@ data: lua_shared_dict: standalone-config: 50m apisix: - proxy_mode: http&stream + proxy_mode: http&stream + node_listen: + - port: 9080 + - port: 9081 stream_proxy: # TCP/UDP proxy tcp: # TCP proxy port list - 9100 @@ -104,6 +107,9 @@ spec: - name: http containerPort: 9080 protocol: TCP + - name: http-alt + containerPort: 9081 + protocol: TCP - name: https containerPort: 9443 protocol: TCP @@ -151,6 +157,10 @@ spec: name: http protocol: TCP targetPort: 9080 + - port: 9081 + name: http-alt + protocol: TCP + targetPort: 9081 - port: {{ .ServiceHTTPSPort }} name: https protocol: TCP diff --git a/test/e2e/framework/manifests/ingress.yaml b/test/e2e/framework/manifests/ingress.yaml index 2c240d7f6c..15fdf6305b 100644 --- a/test/e2e/framework/manifests/ingress.yaml +++ b/test/e2e/framework/manifests/ingress.yaml @@ -291,7 +291,7 @@ data: retry_period: 2s # retry_period is the time in seconds that the acting controller # will wait between tries of actions with the controller. disable: false # Whether to disable leader election. - exec_adc_timeout: 5s + exec_adc_timeout: {{ env "E2E_EXEC_ADC_TIMEOUT" | default "5s" }} provider: type: {{ .ProviderType | default "apisix" }} sync_period: {{ .ProviderSyncPeriod | default "0s" }} diff --git a/test/e2e/gatewayapi/grpcroute.go b/test/e2e/gatewayapi/grpcroute.go index e4485fe08a..01aab5e829 100644 --- a/test/e2e/gatewayapi/grpcroute.go +++ b/test/e2e/gatewayapi/grpcroute.go @@ -19,6 +19,7 @@ package gatewayapi import ( "fmt" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -282,6 +283,162 @@ spec: */ }) + Context("GRPCRoute with sectionName targeting different listeners", func() { + var multiListenerGateway = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: %s +spec: + gatewayClassName: %s + listeners: + - name: http-main + protocol: HTTP + port: 9080 + - name: http-alt + protocol: HTTP + port: 9081 + infrastructure: + parametersRef: + group: apisix.apache.org + kind: GatewayProxy + name: apisix-proxy-config +` + + var routeForMainListener = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: GRPCRoute +metadata: + name: grpc-route-main +spec: + parentRefs: + - name: %s + sectionName: http-main + rules: + - backendRefs: + - name: grpc-infra-backend-v1 + port: 8080 +` + + var routeForAltListener = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: GRPCRoute +metadata: + name: grpc-route-alt +spec: + parentRefs: + - name: %s + sectionName: http-alt + rules: + - backendRefs: + - name: grpc-infra-backend-v1 + port: 8080 +` + + var routeForMainListenerByPort = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: GRPCRoute +metadata: + name: grpc-route-port-main +spec: + parentRefs: + - name: %s + port: 9080 + rules: + - backendRefs: + - name: grpc-infra-backend-v1 + port: 8080 +` + + var routeForAltListenerByPort = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: GRPCRoute +metadata: + name: grpc-route-port-alt +spec: + parentRefs: + - name: %s + port: 9081 + rules: + - backendRefs: + - name: grpc-infra-backend-v1 + port: 8080 +` + + assertRouteReachabilityOnPort := func(port int, shouldSucceed bool) { + check := Eventually(func() error { + return s.RequestEchoBackendOnPort(scaffold.ExpectedResponse{ + EchoRequest: &pb.EchoRequest{}, + }, port) + }).WithTimeout(30 * time.Second).ProbeEvery(time.Second) + if shouldSucceed { + check.ShouldNot(HaveOccurred()) + return + } + check.Should(HaveOccurred()) + } + + runMultiListenerRouteTest := func( + gatewayName string, + routeMainTemplate, routeMainName, routeMainBy string, + routeAltTemplate, routeAltName, routeAltBy string, + deleteMainRouteBy string, + ) { + By("create Gateway with listeners on ports 9080 and 9081") + gateway := fmt.Sprintf(multiListenerGateway, gatewayName, s.Namespace()) + Expect(s.CreateResourceFromString(gateway)).NotTo(HaveOccurred()) + + s.RetryAssertion(func() string { + yaml, _ := s.GetResourceYaml("Gateway", gatewayName) + return yaml + }).Should(ContainSubstring(`status: "True"`)) + + By(routeMainBy) + routeMain := fmt.Sprintf(routeMainTemplate, gatewayName) + s.ResourceApplied("GRPCRoute", routeMainName, routeMain, 1) + + By(routeAltBy) + routeAlt := fmt.Sprintf(routeAltTemplate, gatewayName) + s.ResourceApplied("GRPCRoute", routeAltName, routeAlt, 1) + + By("verify both ports serve traffic before deletion") + assertRouteReachabilityOnPort(9080, true) + assertRouteReachabilityOnPort(9081, true) + + By(deleteMainRouteBy) + Expect(s.DeleteResourceFromString(routeMain)).NotTo(HaveOccurred()) + + assertRouteReachabilityOnPort(9080, false) + assertRouteReachabilityOnPort(9081, true) + } + + It("routes to the configured listener ports when sectionName is set", func() { + runMultiListenerRouteTest( + "grpc-multi-listener", + routeForMainListener, + "grpc-route-main", + "create GRPCRoute targeting listener http-main", + routeForAltListener, + "grpc-route-alt", + "create GRPCRoute targeting listener http-alt", + "delete route for 9080 and verify only 9081 keeps serving traffic", + ) + }) + + It("routes to the configured listener ports when parentRef.port is set", func() { + runMultiListenerRouteTest( + "grpc-multi-listener-by-port", + routeForMainListenerByPort, + "grpc-route-port-main", + "create GRPCRoute targeting port 9080 via parentRef.port", + routeForAltListenerByPort, + "grpc-route-port-alt", + "create GRPCRoute targeting port 9081 via parentRef.port", + "delete route for port 9080 and verify only port 9081 keeps serving traffic", + ) + }) + }) + // TODO: add BackendTrafficPolicy test /* Context("GRPCRoute With BackendTrafficPolicy", func() {}) diff --git a/test/e2e/gatewayapi/httproute.go b/test/e2e/gatewayapi/httproute.go index a00d169328..e3188d31ee 100644 --- a/test/e2e/gatewayapi/httproute.go +++ b/test/e2e/gatewayapi/httproute.go @@ -22,6 +22,8 @@ import ( "crypto/tls" "fmt" "net/http" + "regexp" + "strconv" "strings" "time" @@ -2526,4 +2528,501 @@ spec: Expect(string(msg)).To(Equal(testMessage), "message content verification") }) }) + + Context("HTTPRoute with sectionName targeting different listeners", func() { + // Uses port 9080 (HTTP) and port 9081 (HTTP) + // Both ports are already exposed by the APISIX service + // Uses in-cluster curl to test server_port vars correctly + + var multiListenerGateway = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: %s +spec: + gatewayClassName: %s + listeners: + - name: http-main + protocol: HTTP + port: 9080 + - name: http-alt + protocol: HTTP + port: 9081 + infrastructure: + parametersRef: + group: apisix.apache.org + kind: GatewayProxy + name: apisix-proxy-config +` + + var routeForMainListener = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route-main +spec: + parentRefs: + - name: %s + sectionName: http-main + rules: + - matches: + - path: + type: PathPrefix + value: /get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + + var routeForAltListener = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route-alt +spec: + parentRefs: + - name: %s + sectionName: http-alt + rules: + - matches: + - path: + type: PathPrefix + value: /get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + + var routeNoSectionName = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route-no-section +spec: + parentRefs: + - name: %s + rules: + - matches: + - path: + type: PathPrefix + value: /get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + + var routeInvalidSectionName = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route-invalid-section +spec: + parentRefs: + - name: %s + sectionName: non-existent-listener + rules: + - matches: + - path: + type: PathPrefix + value: /get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + + var routeMultiParentRef = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route-multi-parent +spec: + parentRefs: + - name: %s + sectionName: http-main + - name: %s + sectionName: http-alt + rules: + - matches: + - path: + type: PathPrefix + value: /get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + + var routeForMainListenerByPort = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route-port-main +spec: + parentRefs: + - name: %s + port: 9080 + rules: + - matches: + - path: + type: PathPrefix + value: /get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + + var routeForAltListenerByPort = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route-port-alt +spec: + parentRefs: + - name: %s + port: 9081 + rules: + - matches: + - path: + type: PathPrefix + value: /get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + + var multiListenerGatewayWithHostnames = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: %s +spec: + gatewayClassName: %s + listeners: + - name: http-main + protocol: HTTP + port: 9080 + hostname: api-main.example.com + - name: http-alt + protocol: HTTP + port: 9081 + hostname: api-alt.example.com + infrastructure: + parametersRef: + group: apisix.apache.org + kind: GatewayProxy + name: apisix-proxy-config +` + + var routeNoSectionNameWithHostnames = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route-no-section-hostnames +spec: + parentRefs: + - name: %s + hostnames: + - api-main.example.com + - api-alt.example.com + rules: + - matches: + - path: + type: PathPrefix + value: /get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + + // Get the APISIX service name from the deployer + getApisixServiceName := func() string { + return framework.ProviderType + } + + statusCodePattern := regexp.MustCompile(`\b([1-5][0-9]{2})\b`) + parseHTTPStatusCode := func(output string) (int, error) { + matches := statusCodePattern.FindAllString(strings.TrimSpace(output), -1) + if len(matches) == 0 { + return 0, fmt.Errorf("failed to parse HTTP status code from output: %q", output) + } + + code, err := strconv.Atoi(matches[len(matches)-1]) + if err != nil { + return 0, fmt.Errorf("failed converting status code from output %q: %w", output, err) + } + return code, nil + } + + // Run curl with explicit Host header from within the cluster. + curlInClusterWithHost := func(port int, path, host string) (int, string, error) { + url := fmt.Sprintf("http://%s.%s.svc.cluster.local:%d%s", + getApisixServiceName(), s.Namespace(), port, path) + + args := []string{"-s", "-o", "/dev/null", "-w", "%{http_code}"} + if host != "" { + args = append(args, "-H", fmt.Sprintf("Host: %s", host)) + } + args = append(args, url) + + output, err := s.RunCurlFromK8s(args...) + if err != nil { + return 0, "", err + } + statusCode, err := parseHTTPStatusCode(output) + if err != nil { + return 0, output, err + } + return statusCode, output, nil + } + + // Run curl from within the cluster to the specified port. + curlInCluster := func(port int, path string) (int, string, error) { + return curlInClusterWithHost(port, path, "") + } + + BeforeEach(func() { + By("create GatewayProxy") + Expect(s.CreateResourceFromString(s.GetGatewayProxySpec())).NotTo(HaveOccurred()) + + By("create GatewayClass") + Expect(s.CreateResourceFromString(s.GetGatewayClassYaml())).NotTo(HaveOccurred()) + + s.RetryAssertion(func() string { + yaml, _ := s.GetResourceYaml("GatewayClass", s.Namespace()) + return yaml + }).Should(ContainSubstring(`status: "True"`)) + }) + + It("routes traffic to correct backend based on sectionName (using server_port vars)", func() { + gatewayName := s.Namespace() + + By("create Gateway with two listeners on different ports") + gateway := fmt.Sprintf(multiListenerGateway, gatewayName, s.Namespace()) + Expect(s.CreateResourceFromString(gateway)).NotTo(HaveOccurred()) + + s.RetryAssertion(func() string { + yaml, _ := s.GetResourceYaml("Gateway", gatewayName) + return yaml + }).Should(ContainSubstring(`status: "True"`)) + + By("create HTTPRoute targeting http-main listener (port 9080)") + routeMain := fmt.Sprintf(routeForMainListener, gatewayName) + s.ResourceApplied("HTTPRoute", "route-main", routeMain, 1) + + By("create HTTPRoute targeting http-alt listener (port 9081)") + routeAlt := fmt.Sprintf(routeForAltListener, gatewayName) + s.ResourceApplied("HTTPRoute", "route-alt", routeAlt, 1) + + By("wait for routes to be synced") + time.Sleep(5 * time.Second) + + By("verify route-main is accessible on port 9080 (via in-cluster curl)") + Eventually(func() (int, error) { + statusCode, _, err := curlInCluster(9080, "/get") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK), + "route should be accessible on port 9080") + + By("verify route-alt is accessible on port 9081 (via in-cluster curl)") + Eventually(func() (int, error) { + statusCode, _, err := curlInCluster(9081, "/get") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK), + "route should be accessible on port 9081") + + By("delete route-main and verify route-alt still works") + err := s.DeleteResourceFromString(routeMain) + Expect(err).NotTo(HaveOccurred()) + + // Port 9080 should now return 404 (route deleted) + Eventually(func() (int, error) { + statusCode, _, err := curlInCluster(9080, "/get") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound), + "route should return 404 on port 9080 after deletion") + + // Port 9081 should still return 200 + Eventually(func() (int, error) { + statusCode, _, err := curlInCluster(9081, "/get") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK), + "route should still return 200 on port 9081") + }) + + It("should match all listeners when sectionName is omitted", func() { + gatewayName := s.Namespace() + + By("create Gateway with two listeners") + gateway := fmt.Sprintf(multiListenerGateway, gatewayName, s.Namespace()) + Expect(s.CreateResourceFromString(gateway)).NotTo(HaveOccurred()) + + s.RetryAssertion(func() string { + yaml, _ := s.GetResourceYaml("Gateway", gatewayName) + return yaml + }).Should(ContainSubstring(`status: "True"`)) + + By("create HTTPRoute WITHOUT sectionName") + route := fmt.Sprintf(routeNoSectionName, gatewayName) + s.ResourceApplied("HTTPRoute", "route-no-section", route, 1) + + By("wait for route sync") + time.Sleep(5 * time.Second) + + By("verify route is accessible on port 9080") + Eventually(func() (int, error) { + statusCode, _, err := curlInCluster(9080, "/get") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK), + "route should be accessible on port 9080") + + By("verify route is accessible on port 9081") + Eventually(func() (int, error) { + statusCode, _, err := curlInCluster(9081, "/get") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK), + "route should be accessible on port 9081") + }) + + It("should keep all matched hostnames when sectionName is omitted", func() { + gatewayName := s.Namespace() + + By("create Gateway with two listeners and distinct hostnames") + gateway := fmt.Sprintf(multiListenerGatewayWithHostnames, gatewayName, s.Namespace()) + Expect(s.CreateResourceFromString(gateway)).NotTo(HaveOccurred()) + + s.RetryAssertion(func() string { + yaml, _ := s.GetResourceYaml("Gateway", gatewayName) + return yaml + }).Should(ContainSubstring(`status: "True"`)) + + By("create HTTPRoute WITHOUT sectionName and with both hostnames") + route := fmt.Sprintf(routeNoSectionNameWithHostnames, gatewayName) + s.ResourceApplied("HTTPRoute", "route-no-section-hostnames", route, 1) + + By("verify first hostname is routable") + Eventually(func() (int, error) { + statusCode, _, err := curlInClusterWithHost(9080, "/get", "api-main.example.com") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK), + "api-main.example.com should be routable") + + By("verify second hostname is routable") + Eventually(func() (int, error) { + statusCode, _, err := curlInClusterWithHost(9081, "/get", "api-alt.example.com") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK), + "api-alt.example.com should be routable") + }) + + It("should not route traffic when sectionName references non-existent listener", func() { + gatewayName := s.Namespace() + + By("create Gateway with two listeners") + gateway := fmt.Sprintf(multiListenerGateway, gatewayName, s.Namespace()) + Expect(s.CreateResourceFromString(gateway)).NotTo(HaveOccurred()) + + s.RetryAssertion(func() string { + yaml, _ := s.GetResourceYaml("Gateway", gatewayName) + return yaml + }).Should(ContainSubstring(`status: "True"`)) + + By("create HTTPRoute with invalid sectionName") + route := fmt.Sprintf(routeInvalidSectionName, gatewayName) + Expect(s.CreateResourceFromString(route)).NotTo(HaveOccurred()) + + By("wait for reconciliation") + time.Sleep(5 * time.Second) + + By("verify route is NOT accessible on any port (no matching listener)") + Eventually(func() (int, error) { + statusCode, _, err := curlInCluster(9080, "/get") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound), + "route should not be accessible when sectionName is invalid") + }) + + It("routes traffic to correct backend based on parentRef.port (using server_port vars)", func() { + gatewayName := s.Namespace() + + By("create Gateway with two listeners on different ports") + gateway := fmt.Sprintf(multiListenerGateway, gatewayName, s.Namespace()) + Expect(s.CreateResourceFromString(gateway)).NotTo(HaveOccurred()) + + s.RetryAssertion(func() string { + yaml, _ := s.GetResourceYaml("Gateway", gatewayName) + return yaml + }).Should(ContainSubstring(`status: "True"`)) + + By("create HTTPRoute targeting port 9080 via parentRef.port") + routeMain := fmt.Sprintf(routeForMainListenerByPort, gatewayName) + s.ResourceApplied("HTTPRoute", "route-port-main", routeMain, 1) + + By("create HTTPRoute targeting port 9081 via parentRef.port") + routeAlt := fmt.Sprintf(routeForAltListenerByPort, gatewayName) + s.ResourceApplied("HTTPRoute", "route-port-alt", routeAlt, 1) + + By("verify route-port-main is accessible on port 9080") + Eventually(func() (int, error) { + statusCode, _, err := curlInCluster(9080, "/get") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK), + "route should be accessible on port 9080") + + By("verify route-port-alt is accessible on port 9081") + Eventually(func() (int, error) { + statusCode, _, err := curlInCluster(9081, "/get") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK), + "route should be accessible on port 9081") + + By("delete route-port-main and verify route-port-alt still works") + err := s.DeleteResourceFromString(routeMain) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() (int, error) { + statusCode, _, err := curlInCluster(9080, "/get") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound), + "route should return 404 on port 9080 after deletion") + + Eventually(func() (int, error) { + statusCode, _, err := curlInCluster(9081, "/get") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK), + "route should still return 200 on port 9081") + }) + + It("should route to multiple listeners via multiple parentRefs with sectionName", func() { + gatewayName := s.Namespace() + + By("create Gateway with two listeners") + gateway := fmt.Sprintf(multiListenerGateway, gatewayName, s.Namespace()) + Expect(s.CreateResourceFromString(gateway)).NotTo(HaveOccurred()) + + s.RetryAssertion(func() string { + yaml, _ := s.GetResourceYaml("Gateway", gatewayName) + return yaml + }).Should(ContainSubstring(`status: "True"`)) + + By("create HTTPRoute with multiple parentRefs targeting different listeners") + route := fmt.Sprintf(routeMultiParentRef, gatewayName, gatewayName) + s.ResourceApplied("HTTPRoute", "route-multi-parent", route, 1) + + By("wait for route sync") + time.Sleep(5 * time.Second) + + By("verify route is accessible on port 9080") + Eventually(func() (int, error) { + statusCode, _, err := curlInCluster(9080, "/get") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK), + "route should be accessible on port 9080") + + By("verify route is accessible on port 9081") + Eventually(func() (int, error) { + statusCode, _, err := curlInCluster(9081, "/get") + return statusCode, err + }).WithTimeout(30*time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK), + "route should be accessible on port 9081") + }) + }) }) diff --git a/test/e2e/scaffold/grpc.go b/test/e2e/scaffold/grpc.go index cf79c5134f..66babedaac 100644 --- a/test/e2e/scaffold/grpc.go +++ b/test/e2e/scaffold/grpc.go @@ -21,6 +21,7 @@ import ( "strings" "time" + "github.com/gruntwork-io/terratest/modules/k8s" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" @@ -65,8 +66,24 @@ func (s *Scaffold) DeployGRPCBackend() { } func (s *Scaffold) RequestEchoBackend(exp ExpectedResponse) error { - endpoint := s.apisixTunnels.HTTP.Endpoint() + return s.requestEchoBackendWithEndpoint(exp, s.apisixTunnels.HTTP.Endpoint()) +} + +func (s *Scaffold) RequestEchoBackendOnPort(exp ExpectedResponse, port int) error { + if port == 80 { + return s.RequestEchoBackend(exp) + } + + tunnel := k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, s.dataplaneService.Name, 0, port) + if err := tunnel.ForwardPortE(s.t); err != nil { + return err + } + defer tunnel.Close() + + return s.requestEchoBackendWithEndpoint(exp, tunnel.Endpoint()) +} +func (s *Scaffold) requestEchoBackendWithEndpoint(exp ExpectedResponse, endpoint string) error { endpoint = strings.Replace(endpoint, "localhost", "127.0.0.1", 1) dialOpts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} diff --git a/test/e2e/scaffold/k8s.go b/test/e2e/scaffold/k8s.go index 99fa57db23..30fab2fd4b 100644 --- a/test/e2e/scaffold/k8s.go +++ b/test/e2e/scaffold/k8s.go @@ -254,15 +254,41 @@ func (s *Scaffold) WaitUntilDeploymentAvailable(name string) { } func (s *Scaffold) RunDigDNSClientFromK8s(args ...string) (string, error) { + podName := fmt.Sprintf("dig-test-%d", time.Now().UnixNano()) kubectlArgs := []string{ "run", - "dig", - "-i", + podName, + "--attach=true", "--rm", "--restart=Never", "--image-pull-policy=IfNotPresent", "--image=toolbelt/dig", + "--quiet", + "--command", + "--", + "dig", + } + kubectlArgs = append(kubectlArgs, args...) + return s.RunKubectlAndGetOutput(kubectlArgs...) +} + +// RunCurlFromK8s runs a curl command from a temporary pod inside the cluster. +// This is useful for making HTTP requests from within the cluster, avoiding +// port-forward limitations where server_port variables may not work correctly. +func (s *Scaffold) RunCurlFromK8s(args ...string) (string, error) { + podName := fmt.Sprintf("curl-test-%d", time.Now().UnixNano()) + kubectlArgs := []string{ + "run", + podName, + "--attach=true", + "--rm", + "--restart=Never", + "--image-pull-policy=IfNotPresent", + "--image=alpine/curl:8.17.0", + "--quiet", + "--command", "--", + "curl", } kubectlArgs = append(kubectlArgs, args...) return s.RunKubectlAndGetOutput(kubectlArgs...) diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index af6d828943..d4498d02c7 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -313,6 +313,47 @@ func (s *Scaffold) NewAPISIXClientWithTLSProxy(host string) *httpexpect.Expect { }) } +// NewAPISIXClientForPort creates an HTTP client for a specific APISIX port. +// For built-in ports (80, 443, 9100), it reuses the existing helpers/tunnels. +// For any other port, it creates a new tunnel for that call. +func (s *Scaffold) NewAPISIXClientForPort(port int) (*httpexpect.Expect, error) { + // Check if we can reuse existing tunnels + switch port { + case 80: + return s.NewAPISIXClient(), nil + case 443: + return s.NewAPISIXHttpsClient(""), nil + case 9100: + return s.NewAPISIXClientOnTCPPort(), nil + } + + // Create new tunnel for custom port + serviceName := s.dataplaneService.Name + tunnel := k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, serviceName, 0, port) + if err := tunnel.ForwardPortE(s.t); err != nil { + return nil, fmt.Errorf("failed to create tunnel for port %d: %w", port, err) + } + s.addFinalizers(tunnel.Close) + + u := url.URL{ + Scheme: "http", + Host: tunnel.Endpoint(), + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, + Timeout: 3 * time.Second, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(GinkgoT()), + ), + }), nil +} + func (s *Scaffold) DefaultDataplaneResource() DataplaneResource { return s.Deployer.DefaultDataplaneResource() }