Skip to content

feat(api): per-route port overrides on NebariApp routes #121

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
viniciusdc wants to merge 1 commit into
base: docs/multi-backend-routes-design
Choose a base branch
from
Draft

feat(api): per-route port overrides on NebariApp routes #121

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 32 additions & 13 deletions api/v1/nebariapp_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,23 @@ type NebariAppSpec struct {
// Hostname is the fully qualified domain name where the application should be accessible.
// This will be used to generate HTTPRoute.
// Example: "myapp.nebari.local" or "api.example.com"
//
// Each NebariApp exposes exactly one public hostname and is backed by exactly
// one Kubernetes Service (spec.service). Packs that need multiple hostnames,
// or that genuinely need to fan out to multiple Services, must be split into
// multiple NebariApps. This is an intentional boundary so a NebariApp's TLS,
// auth, landing-page card, and routing concerns all scope to a single
// user-visible URL backed by a single Service. To fan out by path under one
// hostname to different ports on that Service, use routing.routes[].port.
// +kubebuilder:validation:Required
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`
Hostname string `json:"hostname"`

// Service defines the backend Kubernetes Service that should receive traffic.
// Service defines the single backend Kubernetes Service that receives traffic
// for this NebariApp. spec.service.port is the default backend port for routes
// that don't override via routing.routes[].port. Multi-backend (multiple Services
// per NebariApp) is not supported by design — use multiple NebariApps instead.
// +kubebuilder:validation:Required
Service ServiceReference `json:"service"`

Expand Down Expand Up @@ -63,26 +74,24 @@ type NebariAppSpec struct {
LandingPage *LandingPageConfig `json:"landingPage,omitempty"`
}

// ServiceReference identifies the Kubernetes Service that backs this application.
// ServiceReference identifies a Kubernetes Service in the NebariApp's own
// namespace. Cross-namespace backends are not supported: the operator-generated
// HTTPRoute would require a Gateway API ReferenceGrant in the target namespace
// to actually carry traffic, and the operator does not create one. Workloads
// that need to talk across namespaces should do so via in-cluster DNS rather
// than via the public HTTPRoute.
type ServiceReference struct {
// Name is the name of the Kubernetes Service in the same namespace.
// Name is the name of the Kubernetes Service in the NebariApp's namespace.
// +kubebuilder:validation:Required
// +kubebuilder:validation:MinLength=1
Name string `json:"name"`

// Port is the port number on the Service to route traffic to.
// Port is the default port number on the Service to route traffic to.
// Individual routes may override this via routing.routes[].port.
// +kubebuilder:validation:Required
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=65535
Port int32 `json:"port"`

// Namespace is the namespace of the Service (if different from the NebariApp).
// If not specified, defaults to the NebariApp's namespace.
// This allows referencing services in other namespaces for centralized service architectures.
// Note: The operator has cluster-scoped permissions to read Services across all namespaces.
// +optional
// +kubebuilder:validation:MinLength=1
Namespace string `json:"namespace,omitempty"`
}

// RoutingConfig configures routing behavior for the application.
Expand Down Expand Up @@ -125,7 +134,7 @@ type RoutingConfig struct {
// RouteMatch defines a path-based routing rule.
type RouteMatch struct {
// PathPrefix specifies the path prefix to match for routing.
// Traffic matching this prefix will be routed to the service.
// Traffic matching this prefix will be routed to spec.service on the resolved port.
// Must start with "/". Example: "/app-1", "/api/v1"
// +kubebuilder:validation:Required
// +kubebuilder:validation:Pattern=`^/.*`
Expand All @@ -140,6 +149,16 @@ type RouteMatch struct {
// +kubebuilder:validation:Enum=PathPrefix;Exact
// +optional
PathType string `json:"pathType,omitempty"`

// Port optionally overrides the default backend port (spec.service.port)
// for this route. The referenced port must be exposed by spec.service.
// When omitted, the route forwards to spec.service.port. This is the only
// mechanism for path-based port differentiation; per-route backend Services
// are not supported (use multiple NebariApps instead).
// +optional
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=65535
Port *int32 `json:"port,omitempty"`
}

// RoutingTLSConfig controls TLS termination for the HTTPRoute.
Expand Down
13 changes: 11 additions & 2 deletions api/v1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

56 changes: 41 additions & 15 deletions config/crd/bases/reconcilers.nebari.dev_nebariapps.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,14 @@ spec:
Hostname is the fully qualified domain name where the application should be accessible.
This will be used to generate HTTPRoute.
Example: "myapp.nebari.local" or "api.example.com"

Each NebariApp exposes exactly one public hostname and is backed by exactly
one Kubernetes Service (spec.service). Packs that need multiple hostnames,
or that genuinely need to fan out to multiple Services, must be split into
multiple NebariApps. This is an intentional boundary so a NebariApp's TLS,
auth, landing-page card, and routing concerns all scope to a single
user-visible URL backed by a single Service. To fan out by path under one
hostname to different ports on that Service, use routing.routes[].port.
minLength: 1
pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
type: string
Expand Down Expand Up @@ -445,7 +453,7 @@ spec:
pathPrefix:
description: |-
PathPrefix specifies the path prefix to match for routing.
Traffic matching this prefix will be routed to the service.
Traffic matching this prefix will be routed to spec.service on the resolved port.
Must start with "/". Example: "/app-1", "/api/v1"
pattern: ^/.*
type: string
Expand All @@ -461,6 +469,17 @@ spec:
- PathPrefix
- Exact
type: string
port:
description: |-
Port optionally overrides the default backend port (spec.service.port)
for this route. The referenced port must be exposed by spec.service.
When omitted, the route forwards to spec.service.port. This is the only
mechanism for path-based port differentiation; per-route backend Services
are not supported (use multiple NebariApps instead).
format: int32
maximum: 65535
minimum: 1
type: integer
required:
- pathPrefix
type: object
Expand All @@ -477,7 +496,7 @@ spec:
pathPrefix:
description: |-
PathPrefix specifies the path prefix to match for routing.
Traffic matching this prefix will be routed to the service.
Traffic matching this prefix will be routed to spec.service on the resolved port.
Must start with "/". Example: "/app-1", "/api/v1"
pattern: ^/.*
type: string
Expand All @@ -493,6 +512,17 @@ spec:
- PathPrefix
- Exact
type: string
port:
description: |-
Port optionally overrides the default backend port (spec.service.port)
for this route. The referenced port must be exposed by spec.service.
When omitted, the route forwards to spec.service.port. This is the only
mechanism for path-based port differentiation; per-route backend Services
are not supported (use multiple NebariApps instead).
format: int32
maximum: 65535
minimum: 1
type: integer
required:
- pathPrefix
type: object
Expand All @@ -514,25 +544,21 @@ spec:
type: object
type: object
service:
description: Service defines the backend Kubernetes Service that should
receive traffic.
description: |-
Service defines the single backend Kubernetes Service that receives traffic
for this NebariApp. spec.service.port is the default backend port for routes
that don't override via routing.routes[].port. Multi-backend (multiple Services
per NebariApp) is not supported by design — use multiple NebariApps instead.
properties:
name:
description: Name is the name of the Kubernetes Service in the
same namespace.
minLength: 1
type: string
namespace:
description: |-
Namespace is the namespace of the Service (if different from the NebariApp).
If not specified, defaults to the NebariApp's namespace.
This allows referencing services in other namespaces for centralized service architectures.
Note: The operator has cluster-scoped permissions to read Services across all namespaces.
NebariApp's namespace.
minLength: 1
type: string
port:
description: Port is the port number on the Service to route traffic
to.
description: |-
Port is the default port number on the Service to route traffic to.
Individual routes may override this via routing.routes[].port.
format: int32
maximum: 65535
minimum: 1
Expand Down
19 changes: 12 additions & 7 deletions docs/api-reference.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,8 +232,8 @@ _Appears in:_

| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `hostname` _string_ | Hostname is the fully qualified domain name where the application should be accessible.<br />This will be used to generate HTTPRoute.<br />Example: "myapp.nebari.local" or "api.example.com" | | MinLength: 1 <br />Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` <br />Required: \{\} <br /> |
| `service` _[ServiceReference](#servicereference)_ | Service defines the backend Kubernetes Service that should receive traffic. | | Required: \{\} <br /> |
| `hostname` _string_ | Hostname is the fully qualified domain name where the application should be accessible.<br />This will be used to generate HTTPRoute.<br />Example: "myapp.nebari.local" or "api.example.com"<br />Each NebariApp exposes exactly one public hostname and is backed by exactly<br />one Kubernetes Service (spec.service). Packs that need multiple hostnames,<br />or that genuinely need to fan out to multiple Services, must be split into<br />multiple NebariApps. This is an intentional boundary so a NebariApp's TLS,<br />auth, landing-page card, and routing concerns all scope to a single<br />user-visible URL backed by a single Service. To fan out by path under one<br />hostname to different ports on that Service, use routing.routes[].port. | | MinLength: 1 <br />Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` <br />Required: \{\} <br /> |
| `service` _[ServiceReference](#servicereference)_ | Service defines the single backend Kubernetes Service that receives traffic<br />for this NebariApp. spec.service.port is the default backend port for routes<br />that don't override via routing.routes[].port. Multi-backend (multiple Services<br />per NebariApp) is not supported by design — use multiple NebariApps instead. | | Required: \{\} <br /> |
| `routing` _[RoutingConfig](#routingconfig)_ | Routing configures routing behavior including path-based rules and TLS. | | Optional: \{\} <br /> |
| `auth` _[AuthConfig](#authconfig)_ | Auth configures authentication/authorization for the application.<br />When enabled, the application will require OIDC authentication via supporting OIDC Provider. | | Optional: \{\} <br /> |
| `gateway` _string_ | Gateway specifies which shared Gateway to use for routing.<br />Valid values are "public" (default) or "internal". | public | Enum: [public internal] <br />Optional: \{\} <br /> |
Expand Down Expand Up @@ -287,8 +287,9 @@ _Appears in:_

| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `pathPrefix` _string_ | PathPrefix specifies the path prefix to match for routing.<br />Traffic matching this prefix will be routed to the service.<br />Must start with "/". Example: "/app-1", "/api/v1" | | Pattern: `^/.*` <br />Required: \{\} <br /> |
| `pathPrefix` _string_ | PathPrefix specifies the path prefix to match for routing.<br />Traffic matching this prefix will be routed to spec.service on the resolved port.<br />Must start with "/". Example: "/app-1", "/api/v1" | | Pattern: `^/.*` <br />Required: \{\} <br /> |
| `pathType` _string_ | PathType specifies how the path should be matched.<br />Valid values:<br /> - "PathPrefix": Match requests with the specified path prefix<br /> - "Exact": Match requests with the exact path<br />When used in routing.routes, defaults to "PathPrefix".<br />When used in routing.publicRoutes, defaults to "Exact" (safer for auth bypass). | | Enum: [PathPrefix Exact] <br />Optional: \{\} <br /> |
| `port` _integer_ | Port optionally overrides the default backend port (spec.service.port)<br />for this route. The referenced port must be exposed by spec.service.<br />When omitted, the route forwards to spec.service.port. This is the only<br />mechanism for path-based port differentiation; per-route backend Services<br />are not supported (use multiple NebariApps instead). | | Maximum: 65535 <br />Minimum: 1 <br />Optional: \{\} <br /> |


---
Expand Down Expand Up @@ -373,16 +374,20 @@ _Appears in:_

#### ServiceReference

ServiceReference identifies the Kubernetes Service that backs this application.
ServiceReference identifies a Kubernetes Service in the NebariApp's own
namespace. Cross-namespace backends are not supported: the operator-generated
HTTPRoute would require a Gateway API ReferenceGrant in the target namespace
to actually carry traffic, and the operator does not create one. Workloads
that need to talk across namespaces should do so via in-cluster DNS rather
than via the public HTTPRoute.

_Appears in:_
- [NebariAppSpec](#nebariappspec)

| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `name` _string_ | Name is the name of the Kubernetes Service in the same namespace. | | MinLength: 1 <br />Required: \{\} <br /> |
| `port` _integer_ | Port is the port number on the Service to route traffic to. | | Maximum: 65535 <br />Minimum: 1 <br />Required: \{\} <br /> |
| `namespace` _string_ | Namespace is the namespace of the Service (if different from the NebariApp).<br />If not specified, defaults to the NebariApp's namespace.<br />This allows referencing services in other namespaces for centralized service architectures.<br />Note: The operator has cluster-scoped permissions to read Services across all namespaces. | | MinLength: 1 <br />Optional: \{\} <br /> |
| `name` _string_ | Name is the name of the Kubernetes Service in the NebariApp's namespace. | | MinLength: 1 <br />Required: \{\} <br /> |
| `port` _integer_ | Port is the default port number on the Service to route traffic to.<br />Individual routes may override this via routing.routes[].port. | | Maximum: 65535 <br />Minimum: 1 <br />Required: \{\} <br /> |


---
Expand Down
51 changes: 33 additions & 18 deletions internal/controller/reconcilers/core/reconciler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,43 +115,58 @@ func ValidateNamespaceOptIn(ctx context.Context, c client.Client, nebariApp *app
return nil
}

// ValidateService checks if the referenced service exists in the namespace and has the specified port.
// Returns an error if the service doesn't exist or the port is not exposed.
// ValidateService checks that spec.service exists in the NebariApp's namespace
// and exposes both the default port (spec.service.port) and any per-route port
// overrides declared in routing.routes[].port and routing.publicRoutes[].port.
// Cross-namespace backends are not supported; the Service is always looked up
// in the NebariApp's own namespace.
func ValidateService(ctx context.Context, c client.Client, nebariApp *appsv1.NebariApp) error {
service := &corev1.Service{}

// Use specified service namespace, or default to NebariApp's namespace
serviceNamespace := nebariApp.Spec.Service.Namespace
if serviceNamespace == "" {
serviceNamespace = nebariApp.Namespace
}

serviceKey := client.ObjectKey{
Name: nebariApp.Spec.Service.Name,
Namespace: serviceNamespace,
Namespace: nebariApp.Namespace,
}

if err := c.Get(ctx, serviceKey, service); err != nil {
if errors.IsNotFound(err) {
return fmt.Errorf("service %s not found in namespace %s",
nebariApp.Spec.Service.Name, serviceNamespace)
nebariApp.Spec.Service.Name, nebariApp.Namespace)
}
return fmt.Errorf("failed to get service: %w", err)
}

// Validate that the specified port exists on the service
portFound := false
exposed := map[int32]struct{}{}
for _, port := range service.Spec.Ports {
if port.Port == nebariApp.Spec.Service.Port {
portFound = true
break
}
exposed[port.Port] = struct{}{}
}

if !portFound {
if _, ok := exposed[nebariApp.Spec.Service.Port]; !ok {
return fmt.Errorf("service %s does not expose port %d",
nebariApp.Spec.Service.Name, nebariApp.Spec.Service.Port)
}

if nebariApp.Spec.Routing == nil {
return nil
}

for _, route := range nebariApp.Spec.Routing.Routes {
if route.Port == nil {
continue
}
if _, ok := exposed[*route.Port]; !ok {
return fmt.Errorf("route %q: service %s does not expose port %d",
route.PathPrefix, nebariApp.Spec.Service.Name, *route.Port)
}
}
for _, route := range nebariApp.Spec.Routing.PublicRoutes {
if route.Port == nil {
continue
}
if _, ok := exposed[*route.Port]; !ok {
return fmt.Errorf("public route %q: service %s does not expose port %d",
route.PathPrefix, nebariApp.Spec.Service.Name, *route.Port)
}
}

return nil
}
Loading