Skip to content

Commit 3a295fb

Browse files
seanpang-vmwareseanpang-vmware
committed
Add EAS api implementation
1. add a new eas service in nsx-operator 2. register eas service to k8s api server 3. start a http service on 9553 port 4. add eas service implentations for ip usages api yamls
1 parent ff79596 commit 3a295fb

45 files changed

Lines changed: 8874 additions & 398 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.golangci.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,13 @@ linters:
6060
- staticcheck
6161
# v1.EndpointSubset is deprecated in k8s v1.33+; same migration scope as above.
6262
text: "SA1019: v1.EndpointSubset is deprecated"
63+
- linters:
64+
- staticcheck
65+
# kubefake.NewSimpleClientset is deprecated in favour of NewClientset, which
66+
# requires generated apply configurations (--with-applyconfig). Until apply
67+
# configs are generated for this project the existing test helpers cannot be
68+
# migrated, so suppress the warning project-wide.
69+
text: "SA1019: kubefake.NewSimpleClientset is deprecated"
6370

6471
formatters:
6572
enable:

‎Makefile‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,9 @@ changecrd: manifests generate generate-api-docs
4646
manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
4747
$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="github.com/vmware-tanzu/nsx-operator/pkg/apis/legacy/v1alpha1" output:crd:artifacts:config=build/yaml/crd/legacy/
4848
$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" output:crd:artifacts:config=build/yaml/crd/vpc/
49-
$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="github.com/vmware-tanzu/nsx-operator/pkg/apis/eas/v1alpha1" output:crd:artifacts:config=build/yaml/crd/eas/
49+
# NOTE: EAS types are served by the generic API server (via APIService), NOT as CRDs.
50+
# Do NOT run controller-gen crd for pkg/apis/eas/v1alpha1 — installing EAS CRDs alongside
51+
# the APIService causes duplicate-resource conflicts in kube-apiserver.
5052

5153
.PHONY: generate
5254
generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
@@ -97,6 +99,11 @@ build-clean: generate fmt vet ## Build clean binary.
9799
@mkdir -p $(BINDIR)
98100
GOOS=linux go build -o $(BINDIR)/clean $(GOFLAGS) -ldflags '$(LDFLAGS)' cmd_clean/main.go
99101

102+
.PHONY: build-eas
103+
build-eas: generate fmt vet ## Build EAS (Extension API Server) binary.
104+
@mkdir -p $(BINDIR)
105+
GOOS=linux go build -o $(BINDIR)/eas $(GOFLAGS) -ldflags '$(LDFLAGS)' cmd_eas/main.go
106+
100107
.PHONY: run
101108
run: manifests generate fmt vet ## Run a controller from your host.
102109
go run ./cmd/main.go
@@ -113,6 +120,10 @@ docker-push: ## Push docker image with the manager.
113120
photon:
114121
docker build -t github.com/vmware-tanzu/nsx-operator -f build/image/photon/Dockerfile .
115122

123+
.PHONY: eas
124+
eas:
125+
docker build -t github.com/vmware-tanzu/nsx-eas -f build/image/eas/Dockerfile .
126+
116127
.PHONY: clean
117128
clean:
118129
@rm -rf $(BINDIR)
@@ -164,6 +175,28 @@ code-generator: ## Download code-generator locally if necessary.
164175
generated: code-generator
165176
./hack/update-codegen.sh
166177

178+
OPENAPI_GEN = $(shell pwd)/bin/openapi-gen
179+
# openapi-gen moved from k8s.io/code-generator to k8s.io/kube-openapi (since ~v0.27).
180+
# We install it directly via "go install" in the project root so it uses the
181+
# version already pinned in go.mod without needing to create a throw-away module.
182+
.PHONY: openapi-gen
183+
openapi-gen: ## Build openapi-gen binary into bin/ (uses k8s.io/kube-openapi version from go.mod).
184+
@[ -f $(OPENAPI_GEN) ] || GOBIN=$(CURDIR)/bin go install k8s.io/kube-openapi/cmd/openapi-gen
185+
186+
.PHONY: generate-eas-openapi
187+
generate-eas-openapi: openapi-gen ## Generate OpenAPI definitions for EAS types plus the apimachinery meta/v1 and version types they reference.
188+
$(OPENAPI_GEN) \
189+
--output-dir pkg/apis/eas/v1alpha1 \
190+
--output-pkg github.com/vmware-tanzu/nsx-operator/pkg/apis/eas/v1alpha1 \
191+
--output-file zz_generated.openapi.go \
192+
--go-header-file hack/boilerplate.go.txt \
193+
--report-filename - \
194+
github.com/vmware-tanzu/nsx-operator/pkg/apis/eas/v1alpha1 \
195+
k8s.io/apimachinery/pkg/apis/meta/v1 \
196+
k8s.io/apimachinery/pkg/version
197+
@echo ""
198+
@echo "Generated pkg/apis/eas/v1alpha1/zz_generated.openapi.go"
199+
167200
CRD_REF_DOCS = $(shell pwd)/bin/crd-ref-docs
168201
.PHONY: crd-ref-docs
169202
crd-ref-docs: ## Install crd-ref-docs

‎build/image/eas/Dockerfile‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
FROM golang:1.25.7 as golang-build
2+
3+
WORKDIR /source
4+
5+
COPY . /source
6+
RUN CGO_ENABLED=0 go build -o eas cmd_eas/main.go
7+
8+
FROM photon
9+
10+
RUN tdnf -y install shadow && \
11+
useradd -s /bin/bash eas
12+
13+
COPY --from=golang-build /source/eas /usr/local/bin/
14+
15+
USER eas
16+
17+
ENTRYPOINT ["/usr/local/bin/eas"]

‎build/yaml/crd/eas/eas.nsx.vmware.com_vpcipaddressusages.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,9 @@ spec:
107107
- start
108108
type: object
109109
type: array
110-
name:
110+
ipBlockName:
111+
description: IPBlockName is the NSX resource ID of the IP block
112+
(the leaf segment of the policy path).
111113
type: string
112114
percentageUsed:
113115
description: Percentage of used IP address space.

‎build/yaml/eas/apiservice.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,4 @@ spec:
1010
service:
1111
name: nsx-eas
1212
namespace: vmware-system-nsx
13-
insecureSkipTLSVerify: true
13+
insecureSkipTLSVerify: true

‎build/yaml/eas/clusterrole.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ rules:
1010
- subnetippools
1111
- subnetdhcpserverstats
1212
verbs: ["get", "list"]
13+
- apiGroups: ["apiregistration.k8s.io"]
14+
resources: ["apiservices"]
15+
verbs: ["get", "create", "update", "patch"]
1316
---
1417
apiVersion: rbac.authorization.k8s.io/v1
1518
kind: ClusterRoleBinding

‎build/yaml/eas/eas-service.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: nsx-eas
5+
namespace: vmware-system-nsx
6+
labels:
7+
app.kubernetes.io/name: service
8+
app.kubernetes.io/instance: eas-service
9+
app.kubernetes.io/component: eas
10+
app.kubernetes.io/created-by: nsx-operator
11+
app.kubernetes.io/part-of: nsx-operator
12+
app.kubernetes.io/managed-by: kustomize
13+
spec:
14+
ports:
15+
- port: 443
16+
targetPort: 9553
17+
protocol: TCP
18+
selector:
19+
component: nsx-ncp

‎build/yaml/eas/service.yaml‎

Lines changed: 0 additions & 12 deletions
This file was deleted.

‎cmd_eas/main.go‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
/* Copyright © 2026 Broadcom, Inc. All Rights Reserved.
2+
SPDX-License-Identifier: Apache-2.0 */
3+
4+
package main
5+
6+
import (
7+
"fmt"
8+
"os"
9+
"time"
10+
11+
"k8s.io/apimachinery/pkg/runtime"
12+
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
13+
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
14+
ctrl "sigs.k8s.io/controller-runtime"
15+
logf "sigs.k8s.io/controller-runtime/pkg/log"
16+
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
17+
18+
vpcv1alpha1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1"
19+
"github.com/vmware-tanzu/nsx-operator/pkg/config"
20+
"github.com/vmware-tanzu/nsx-operator/pkg/eas"
21+
"github.com/vmware-tanzu/nsx-operator/pkg/eas/server"
22+
"github.com/vmware-tanzu/nsx-operator/pkg/logger"
23+
"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
24+
pkgutil "github.com/vmware-tanzu/nsx-operator/pkg/util"
25+
)
26+
27+
var (
28+
// scheme is the controller-runtime manager's scheme.
29+
// It only needs types that the VPC info provider and EAS storage layer
30+
// read from kube-apiserver (VPCNetworkConfiguration, Subnet …).
31+
// EAS API types (VPCIPAddressUsage etc.) are served entirely from NSX data
32+
// and live in the generic API server's own scheme (pkg/eas/server/scheme.go).
33+
scheme = runtime.NewScheme()
34+
)
35+
36+
func init() {
37+
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
38+
utilruntime.Must(vpcv1alpha1.AddToScheme(scheme))
39+
}
40+
41+
func main() {
42+
// config.AddFlags registers all flags (including -nsxconfig and -log-level)
43+
// and calls flag.Parse() internally.
44+
config.AddFlags()
45+
46+
cf, err := config.NewNSXOperatorConfigFromFile()
47+
if err != nil {
48+
fmt.Fprintf(os.Stderr, "Failed to read config: %v\n", err)
49+
os.Exit(1)
50+
}
51+
52+
log := logger.ZapCustomLogger(cf.DefaultConfig.Debug, config.LogLevel)
53+
logger.Log = log
54+
// Register logger with controller-runtime to suppress "log.SetLogger(...) was never called" warning.
55+
logf.SetLogger(log.Logger)
56+
57+
log.Info("Starting NSX Extension API Server")
58+
59+
// Generate TLS certificates for the EAS HTTPS server.
60+
// The generic API server (k8s.io/apiserver) uses dynamic certificate loading
61+
// via dynamiccertificates.NewDynamicServingContentFromFiles, so it
62+
// automatically picks up renewed cert files without a pod restart.
63+
// The returned CA PEM is injected into the APIService caBundle so that
64+
// kube-apiserver can verify the EAS TLS connection.
65+
caCert, err := pkgutil.GenerateEASCerts()
66+
if err != nil {
67+
log.Error(err, "Failed to generate EAS certificates")
68+
os.Exit(1)
69+
}
70+
log.Info("EAS certificates generated successfully")
71+
go refreshEASCertPeriodically()
72+
73+
// Initialize NSX client.
74+
nsxClient := nsx.GetClient(cf)
75+
if nsxClient == nil {
76+
log.Error(nil, "Failed to get NSX client")
77+
os.Exit(1)
78+
}
79+
log.Info("NSX client initialized")
80+
81+
// Build the k8s rest config and a controller-runtime manager.
82+
// EAS is read-only so leader election is disabled — all replicas serve
83+
// concurrently, load-balanced by the kube Service.
84+
cfg, err := pkgutil.GetConfig()
85+
if err != nil {
86+
log.Error(err, "Failed to get REST config for manager")
87+
os.Exit(1)
88+
}
89+
90+
// Health and readiness probes are served by the generic API server on the
91+
// EAS port (default 9553) at /healthz and /readyz over HTTPS.
92+
// The NSX connectivity check is wired into /readyz via EASServer so that
93+
// the pod is removed from Service endpoints when NSX is unreachable.
94+
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
95+
Scheme: scheme,
96+
LeaderElection: false,
97+
// Metrics and health probes are both handled by the generic API server;
98+
// disable the controller-runtime sidecars to avoid unused open ports.
99+
Metrics: metricsserver.Options{BindAddress: "0"},
100+
HealthProbeBindAddress: "0",
101+
})
102+
if err != nil {
103+
log.Error(err, "Failed to create manager")
104+
os.Exit(1)
105+
}
106+
107+
vpcProvider := eas.NewK8sVPCInfoProvider(mgr.GetClient())
108+
srv := server.NewEASServer(nsxClient, vpcProvider, mgr.GetClient(), cfg, caCert)
109+
if err := mgr.Add(srv); err != nil {
110+
log.Error(err, "Failed to add EAS server to manager")
111+
os.Exit(1)
112+
}
113+
114+
log.Info("Starting manager")
115+
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
116+
log.Error(err, "Failed to start manager")
117+
os.Exit(1)
118+
}
119+
}
120+
121+
// refreshEASCertPeriodically regenerates the EAS TLS certificate every 30 days.
122+
// The generic API server loads certs via dynamiccertificates.NewDynamicServingContentFromFiles,
123+
// which watches the cert files for changes and reloads them automatically — so new
124+
// certs are picked up without a pod restart.
125+
func refreshEASCertPeriodically() {
126+
ticker := time.NewTicker(30 * 24 * time.Hour)
127+
defer ticker.Stop()
128+
for range ticker.C {
129+
logger.Log.Info("Refreshing EAS certificates...")
130+
if _, err := pkgutil.GenerateEASCerts(); err != nil {
131+
logger.Log.Error(err, "Failed to refresh EAS certificates")
132+
} else {
133+
logger.Log.Info("EAS certificates refreshed successfully")
134+
}
135+
}
136+
}

‎go.mod‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ require (
4545
gopkg.in/yaml.v2 v2.4.0
4646
k8s.io/api v0.35.1
4747
k8s.io/apimachinery v0.35.1
48+
k8s.io/apiserver v0.35.1
4849
k8s.io/client-go v0.35.1
4950
k8s.io/code-generator v0.35.1
5051
k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2
@@ -58,16 +59,25 @@ require (
5859
)
5960

6061
require (
62+
cel.dev/expr v0.24.0 // indirect
63+
github.com/NYTimes/gziphandler v1.1.1 // indirect
64+
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
6165
github.com/beevik/etree v1.6.0 // indirect
6266
github.com/beorn7/perks v1.0.1 // indirect
67+
github.com/blang/semver/v4 v4.0.0 // indirect
68+
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
6369
github.com/cespare/xxhash/v2 v2.3.0 // indirect
70+
github.com/coreos/go-semver v0.3.1 // indirect
71+
github.com/coreos/go-systemd/v22 v22.7.0 // indirect
6472
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
6573
github.com/emicklei/go-restful/v3 v3.13.0 // indirect
6674
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
6775
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
76+
github.com/felixge/httpsnoop v1.0.4 // indirect
6877
github.com/fsnotify/fsnotify v1.9.0 // indirect
6978
github.com/fxamacker/cbor/v2 v2.9.1 // indirect
7079
github.com/gibson042/canonicaljson-go v1.0.3 // indirect
80+
github.com/go-logr/stdr v1.2.2 // indirect
7181
github.com/go-logr/zapr v1.3.0 // indirect
7282
github.com/go-openapi/jsonpointer v0.23.1 // indirect
7383
github.com/go-openapi/jsonreference v0.21.5 // indirect
@@ -83,11 +93,18 @@ require (
8393
github.com/go-openapi/swag/stringutils v0.26.0 // indirect
8494
github.com/go-openapi/swag/typeutils v0.26.0 // indirect
8595
github.com/go-openapi/swag/yamlutils v0.26.0 // indirect
96+
github.com/gogo/protobuf v1.3.2 // indirect
97+
github.com/golang/protobuf v1.5.4 // indirect
8698
github.com/google/btree v1.1.3 // indirect
99+
github.com/google/cel-go v0.26.0 // indirect
87100
github.com/google/gnostic-models v0.7.0 // indirect
88101
github.com/google/go-cmp v0.7.0 // indirect
89102
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
103+
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
104+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect
105+
github.com/inconshreveable/mousetrap v1.1.0 // indirect
90106
github.com/json-iterator/go v1.1.12 // indirect
107+
github.com/kylelemons/godebug v1.1.0 // indirect
91108
github.com/mattn/go-colorable v0.1.14 // indirect
92109
github.com/mattn/go-isatty v0.0.21 // indirect
93110
github.com/moby/spdystream v0.5.1 // indirect
@@ -101,12 +118,28 @@ require (
101118
github.com/prometheus/client_model v0.6.2 // indirect
102119
github.com/prometheus/common v0.67.5 // indirect
103120
github.com/prometheus/procfs v0.20.1 // indirect
121+
github.com/spf13/cobra v1.10.0 // indirect
104122
github.com/spf13/pflag v1.0.10 // indirect
123+
github.com/stoewer/go-strcase v1.3.0 // indirect
105124
github.com/stretchr/objx v0.5.2 // indirect
106125
github.com/x448/float16 v0.8.4 // indirect
126+
go.etcd.io/etcd/api/v3 v3.6.5 // indirect
127+
go.etcd.io/etcd/client/pkg/v3 v3.6.5 // indirect
128+
go.etcd.io/etcd/client/v3 v3.6.5 // indirect
129+
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
130+
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
131+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
132+
go.opentelemetry.io/otel v1.36.0 // indirect
133+
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect
134+
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect
135+
go.opentelemetry.io/otel/metric v1.36.0 // indirect
136+
go.opentelemetry.io/otel/sdk v1.36.0 // indirect
137+
go.opentelemetry.io/otel/trace v1.36.0 // indirect
138+
go.opentelemetry.io/proto/otlp v1.5.0 // indirect
107139
go.uber.org/multierr v1.11.0 // indirect
108140
go.yaml.in/yaml/v2 v2.4.4 // indirect
109141
go.yaml.in/yaml/v3 v3.0.4 // indirect
142+
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
110143
golang.org/x/mod v0.35.0 // indirect
111144
golang.org/x/net v0.53.0 // indirect
112145
golang.org/x/oauth2 v0.36.0 // indirect
@@ -115,14 +148,21 @@ require (
115148
golang.org/x/text v0.36.0 // indirect
116149
golang.org/x/tools v0.44.0 // indirect
117150
gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect
151+
google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect
152+
google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect
153+
google.golang.org/grpc v1.72.2 // indirect
118154
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
119155
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
120156
gopkg.in/inf.v0 v0.9.1 // indirect
157+
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
121158
gopkg.in/yaml.v3 v3.0.1 // indirect
122159
k8s.io/apiextensions-apiserver v0.35.0 // indirect
160+
k8s.io/component-base v0.35.1 // indirect
123161
k8s.io/gengo/v2 v2.0.0-20260408192533-25e2208e0dc3 // indirect
124162
k8s.io/klog/v2 v2.140.0 // indirect
163+
k8s.io/kms v0.35.1 // indirect
125164
k8s.io/kube-openapi v0.0.0-20260414162039-ec9c827d403f // indirect
165+
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect
126166
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
127167
sigs.k8s.io/randfill v1.0.0 // indirect
128168
sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect

0 commit comments

Comments
 (0)