Skip to content

Commit 06895a8

Browse files
seanpang-vmwareseanpang-vmware
committed
Add EAS api implementation
1. add a new eas service in nsx-operator 2. register eas service to k8s api server 3. start a http service on 9553 port 4. add eas service implentations for ip usages api yamls
1 parent 73f6e84 commit 06895a8

36 files changed

Lines changed: 4504 additions & 288 deletions

‎Makefile‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,11 @@ build-clean: generate fmt vet ## Build clean binary.
9797
@mkdir -p $(BINDIR)
9898
GOOS=linux go build -o $(BINDIR)/clean $(GOFLAGS) -ldflags '$(LDFLAGS)' cmd_clean/main.go
9999

100+
.PHONY: build-eas
101+
build-eas: generate fmt vet ## Build EAS (Extension API Server) binary.
102+
@mkdir -p $(BINDIR)
103+
GOOS=linux go build -o $(BINDIR)/eas $(GOFLAGS) -ldflags '$(LDFLAGS)' cmd_eas/main.go
104+
100105
.PHONY: run
101106
run: manifests generate fmt vet ## Run a controller from your host.
102107
go run ./cmd/main.go
@@ -113,6 +118,10 @@ docker-push: ## Push docker image with the manager.
113118
photon:
114119
docker build -t github.com/vmware-tanzu/nsx-operator -f build/image/photon/Dockerfile .
115120

121+
.PHONY: eas
122+
eas:
123+
docker build -t github.com/vmware-tanzu/nsx-eas -f build/image/eas/Dockerfile .
124+
116125
.PHONY: clean
117126
clean:
118127
@rm -rf $(BINDIR)

‎build/image/eas/Dockerfile‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
FROM golang:1.25.7 as golang-build
2+
3+
WORKDIR /source
4+
5+
COPY . /source
6+
RUN CGO_ENABLED=0 go build -o eas cmd_eas/main.go
7+
8+
FROM photon
9+
10+
RUN tdnf -y install shadow && \
11+
useradd -s /bin/bash eas
12+
13+
COPY --from=golang-build /source/eas /usr/local/bin/
14+
15+
USER eas
16+
17+
ENTRYPOINT ["/usr/local/bin/eas"]

‎build/yaml/crd/eas/eas.nsx.vmware.com_vpcipaddressusages.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,9 @@ spec:
107107
- start
108108
type: object
109109
type: array
110-
name:
110+
ipBlockName:
111+
description: IPBlockName is the NSX resource ID of the IP block
112+
(the leaf segment of the policy path).
111113
type: string
112114
percentageUsed:
113115
description: Percentage of used IP address space.

‎build/yaml/eas/apiservice.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,4 @@ spec:
1010
service:
1111
name: nsx-eas
1212
namespace: vmware-system-nsx
13-
insecureSkipTLSVerify: true
13+
insecureSkipTLSVerify: true

‎build/yaml/eas/clusterrole.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ rules:
1010
- subnetippools
1111
- subnetdhcpserverstats
1212
verbs: ["get", "list"]
13+
- apiGroups: ["apiregistration.k8s.io"]
14+
resources: ["apiservices"]
15+
verbs: ["get", "create", "update", "patch"]
1316
---
1417
apiVersion: rbac.authorization.k8s.io/v1
1518
kind: ClusterRoleBinding

‎build/yaml/eas/eas-service.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: nsx-eas
5+
namespace: vmware-system-nsx
6+
labels:
7+
app.kubernetes.io/name: service
8+
app.kubernetes.io/instance: eas-service
9+
app.kubernetes.io/component: eas
10+
app.kubernetes.io/created-by: nsx-operator
11+
app.kubernetes.io/part-of: nsx-operator
12+
app.kubernetes.io/managed-by: kustomize
13+
spec:
14+
ports:
15+
- port: 443
16+
targetPort: 9553
17+
protocol: TCP
18+
selector:
19+
component: nsx-ncp

‎build/yaml/eas/service.yaml‎

Lines changed: 0 additions & 12 deletions
This file was deleted.

‎cmd_eas/main.go‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
/* Copyright © 2026 Broadcom, Inc. All Rights Reserved.
2+
SPDX-License-Identifier: Apache-2.0 */
3+
4+
package main
5+
6+
import (
7+
"flag"
8+
"fmt"
9+
"os"
10+
"time"
11+
12+
"k8s.io/apimachinery/pkg/runtime"
13+
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
14+
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
15+
ctrl "sigs.k8s.io/controller-runtime"
16+
logf "sigs.k8s.io/controller-runtime/pkg/log"
17+
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
18+
19+
vpcv1alpha1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1"
20+
"github.com/vmware-tanzu/nsx-operator/pkg/config"
21+
"github.com/vmware-tanzu/nsx-operator/pkg/eas"
22+
"github.com/vmware-tanzu/nsx-operator/pkg/eas/server"
23+
"github.com/vmware-tanzu/nsx-operator/pkg/logger"
24+
"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
25+
pkgutil "github.com/vmware-tanzu/nsx-operator/pkg/util"
26+
)
27+
28+
var (
29+
// scheme is the controller-runtime manager's scheme.
30+
// It only needs types that the VPC info provider and EAS storage layer
31+
// read from kube-apiserver (VPCNetworkConfiguration, Subnet …).
32+
// EAS API types (VPCIPAddressUsage etc.) are served entirely from NSX data
33+
// and live in the generic API server's own scheme (pkg/eas/server/scheme.go).
34+
scheme = runtime.NewScheme()
35+
configFilePath string
36+
)
37+
38+
func init() {
39+
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
40+
utilruntime.Must(vpcv1alpha1.AddToScheme(scheme))
41+
}
42+
43+
func main() {
44+
var logLevel int
45+
flag.StringVar(&configFilePath, "nsxconfig", "/etc/nsx-ujo/ncp.ini", "NSX Operator configuration file path")
46+
flag.IntVar(&logLevel, "log-level", 0, "Log verbosity level (0=info, 1=debug, 2=trace)")
47+
flag.StringVar(&config.ProbeAddr, "health-probe-bind-address", ":8385", "The address the probe endpoint binds to.")
48+
flag.Parse()
49+
50+
config.UpdateConfigFilePath(configFilePath)
51+
cf, err := config.NewNSXOperatorConfigFromFile()
52+
if err != nil {
53+
fmt.Fprintf(os.Stderr, "Failed to read config: %v\n", err)
54+
os.Exit(1)
55+
}
56+
57+
log := logger.ZapCustomLogger(cf.DefaultConfig.Debug, logLevel)
58+
logger.Log = log
59+
// Register logger with controller-runtime to suppress "log.SetLogger(...) was never called" warning.
60+
logf.SetLogger(log.Logger)
61+
62+
log.Info("Starting NSX Extension API Server")
63+
64+
// Generate TLS certificates for the EAS HTTPS server.
65+
// The generic API server (k8s.io/apiserver) uses dynamic certificate loading
66+
// via dynamiccertificates.NewDynamicServingContentFromFiles, so it
67+
// automatically picks up renewed cert files without a pod restart.
68+
// The returned CA PEM is injected into the APIService caBundle so that
69+
// kube-apiserver can verify the EAS TLS connection.
70+
caCert, err := pkgutil.GenerateEASCerts()
71+
if err != nil {
72+
log.Error(err, "Failed to generate EAS certificates")
73+
os.Exit(1)
74+
}
75+
log.Info("EAS certificates generated successfully")
76+
go refreshEASCertPeriodically()
77+
78+
// Initialize NSX client.
79+
nsxClient := nsx.GetClient(cf)
80+
if nsxClient == nil {
81+
log.Error(nil, "Failed to get NSX client")
82+
os.Exit(1)
83+
}
84+
log.Info("NSX client initialized")
85+
86+
// Build the k8s rest config and a controller-runtime manager.
87+
// EAS is read-only so leader election is disabled — all replicas serve
88+
// concurrently, load-balanced by the kube Service.
89+
cfg, err := pkgutil.GetConfig()
90+
if err != nil {
91+
log.Error(err, "Failed to get REST config for manager")
92+
os.Exit(1)
93+
}
94+
95+
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
96+
Scheme: scheme,
97+
LeaderElection: false,
98+
// Metrics are served by the generic API server at /metrics on the EAS port.
99+
// Disable the controller-runtime metrics server to avoid port conflicts.
100+
Metrics: metricsserver.Options{BindAddress: "0"},
101+
// Liveness/readiness probes for the pod (distinct from the generic API
102+
// server's built-in /healthz, /readyz, /livez endpoints on the EAS port).
103+
HealthProbeBindAddress: config.ProbeAddr,
104+
})
105+
if err != nil {
106+
log.Error(err, "Failed to create manager")
107+
os.Exit(1)
108+
}
109+
110+
vpcProvider := eas.NewK8sVPCInfoProvider(mgr.GetClient())
111+
srv := server.NewEASServer(nsxClient, vpcProvider, mgr.GetClient(), cfg, caCert)
112+
if err := mgr.Add(srv); err != nil {
113+
log.Error(err, "Failed to add EAS server to manager")
114+
os.Exit(1)
115+
}
116+
117+
if err := mgr.AddHealthzCheck("healthz", nsxClient.NSXChecker.CheckNSXHealth); err != nil {
118+
log.Error(err, "Failed to set up health check")
119+
os.Exit(1)
120+
}
121+
if err := mgr.AddReadyzCheck("readyz", nsxClient.NSXChecker.CheckNSXHealth); err != nil {
122+
log.Error(err, "Failed to set up ready check")
123+
os.Exit(1)
124+
}
125+
126+
log.Info("Starting manager")
127+
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
128+
log.Error(err, "Failed to start manager")
129+
os.Exit(1)
130+
}
131+
}
132+
133+
// refreshEASCertPeriodically regenerates the EAS TLS certificate every 30 days.
134+
// The generic API server loads certs via dynamiccertificates.NewDynamicServingContentFromFiles,
135+
// which watches the cert files for changes and reloads them automatically — so new
136+
// certs are picked up without a pod restart.
137+
func refreshEASCertPeriodically() {
138+
ticker := time.NewTicker(30 * 24 * time.Hour)
139+
defer ticker.Stop()
140+
for range ticker.C {
141+
logger.Log.Info("Refreshing EAS certificates...")
142+
if _, err := pkgutil.GenerateEASCerts(); err != nil {
143+
logger.Log.Error(err, "Failed to refresh EAS certificates")
144+
} else {
145+
logger.Log.Info("EAS certificates refreshed successfully")
146+
}
147+
}
148+
}

‎go.mod‎

Lines changed: 46 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ require (
2121
github.com/google/uuid v1.6.0
2222
github.com/kevinburke/ssh_config v1.2.0
2323
github.com/openlyinc/pointy v1.1.2
24-
github.com/prometheus/client_golang v1.19.1
24+
github.com/prometheus/client_golang v1.23.2
2525
github.com/rs/zerolog v1.34.0
2626
github.com/sirupsen/logrus v1.9.3
2727
github.com/stretchr/testify v1.11.1
@@ -35,41 +35,58 @@ require (
3535
github.com/vmware/vsphere-automation-sdk-go/services/nsxt v0.0.0-20260310075027-d32fca6a7b22
3636
github.com/vmware/vsphere-automation-sdk-go/services/nsxt-mp v0.0.0-20260310075027-d32fca6a7b22
3737
go.uber.org/automaxprocs v1.5.3
38-
go.uber.org/zap v1.26.0
38+
go.uber.org/zap v1.27.0
3939
golang.org/x/crypto v0.47.0
4040
golang.org/x/sync v0.19.0
4141
golang.org/x/time v0.14.0
4242
gopkg.in/ini.v1 v1.66.4
4343
gopkg.in/yaml.v2 v2.4.0
4444
k8s.io/api v0.35.1
4545
k8s.io/apimachinery v0.35.1
46+
k8s.io/apiserver v0.35.1
4647
k8s.io/client-go v0.35.1
4748
k8s.io/code-generator v0.31.0
4849
k8s.io/utils v0.0.0-20260108192941-914a6e750570
4950
sigs.k8s.io/controller-runtime v0.19.0
5051
)
5152

5253
require (
54+
cel.dev/expr v0.24.0 // indirect
55+
github.com/NYTimes/gziphandler v1.1.1 // indirect
56+
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
5357
github.com/beevik/etree v1.1.0 // indirect
5458
github.com/beorn7/perks v1.0.1 // indirect
59+
github.com/blang/semver/v4 v4.0.0 // indirect
60+
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
5561
github.com/cespare/xxhash/v2 v2.3.0 // indirect
62+
github.com/coreos/go-semver v0.3.1 // indirect
63+
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
5664
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
5765
github.com/emicklei/go-restful/v3 v3.13.0 // indirect
5866
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
5967
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
60-
github.com/fsnotify/fsnotify v1.7.0 // indirect
68+
github.com/felixge/httpsnoop v1.0.4 // indirect
69+
github.com/fsnotify/fsnotify v1.9.0 // indirect
6170
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
6271
github.com/gibson042/canonicaljson-go v1.0.3 // indirect
72+
github.com/go-logr/stdr v1.2.2 // indirect
6373
github.com/go-logr/zapr v1.3.0 // indirect
6474
github.com/go-openapi/jsonpointer v0.21.2 // indirect
6575
github.com/go-openapi/jsonreference v0.21.0 // indirect
6676
github.com/go-openapi/swag v0.23.1 // indirect
6777
github.com/gogo/protobuf v1.3.2 // indirect
78+
github.com/golang/protobuf v1.5.4 // indirect
79+
github.com/google/btree v1.1.3 // indirect
80+
github.com/google/cel-go v0.26.0 // indirect
6881
github.com/google/gnostic-models v0.7.0 // indirect
6982
github.com/google/go-cmp v0.7.0 // indirect
7083
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
84+
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
85+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect
86+
github.com/inconshreveable/mousetrap v1.1.0 // indirect
7187
github.com/josharian/intern v1.0.0 // indirect
7288
github.com/json-iterator/go v1.1.12 // indirect
89+
github.com/kylelemons/godebug v1.1.0 // indirect
7390
github.com/mailru/easyjson v0.9.1 // indirect
7491
github.com/mattn/go-colorable v0.1.13 // indirect
7592
github.com/mattn/go-isatty v0.0.19 // indirect
@@ -82,16 +99,31 @@ require (
8299
github.com/onsi/gomega v1.39.1 // indirect
83100
github.com/pkg/errors v0.9.1 // indirect
84101
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
85-
github.com/prometheus/client_model v0.6.1 // indirect
86-
github.com/prometheus/common v0.55.0 // indirect
87-
github.com/prometheus/procfs v0.15.1 // indirect
102+
github.com/prometheus/client_model v0.6.2 // indirect
103+
github.com/prometheus/common v0.66.1 // indirect
104+
github.com/prometheus/procfs v0.16.1 // indirect
105+
github.com/spf13/cobra v1.10.0 // indirect
88106
github.com/spf13/pflag v1.0.10 // indirect
107+
github.com/stoewer/go-strcase v1.3.0 // indirect
89108
github.com/stretchr/objx v0.5.2 // indirect
90109
github.com/x448/float16 v0.8.4 // indirect
110+
go.etcd.io/etcd/api/v3 v3.6.5 // indirect
111+
go.etcd.io/etcd/client/pkg/v3 v3.6.5 // indirect
112+
go.etcd.io/etcd/client/v3 v3.6.5 // indirect
113+
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
114+
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
115+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
116+
go.opentelemetry.io/otel v1.36.0 // indirect
117+
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect
118+
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect
119+
go.opentelemetry.io/otel/metric v1.36.0 // indirect
120+
go.opentelemetry.io/otel/sdk v1.36.0 // indirect
121+
go.opentelemetry.io/otel/trace v1.36.0 // indirect
122+
go.opentelemetry.io/proto/otlp v1.5.0 // indirect
91123
go.uber.org/multierr v1.11.0 // indirect
92124
go.yaml.in/yaml/v2 v2.4.3 // indirect
93125
go.yaml.in/yaml/v3 v3.0.4 // indirect
94-
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
126+
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
95127
golang.org/x/mod v0.32.0 // indirect
96128
golang.org/x/net v0.49.0 // indirect
97129
golang.org/x/oauth2 v0.34.0 // indirect
@@ -100,14 +132,21 @@ require (
100132
golang.org/x/text v0.33.0 // indirect
101133
golang.org/x/tools v0.41.0 // indirect
102134
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
135+
google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect
136+
google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect
137+
google.golang.org/grpc v1.72.2 // indirect
103138
google.golang.org/protobuf v1.36.11 // indirect
104139
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
105140
gopkg.in/inf.v0 v0.9.1 // indirect
141+
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
106142
gopkg.in/yaml.v3 v3.0.1 // indirect
107143
k8s.io/apiextensions-apiserver v0.31.0 // indirect
144+
k8s.io/component-base v0.35.1 // indirect
108145
k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f // indirect
109146
k8s.io/klog/v2 v2.130.1 // indirect
147+
k8s.io/kms v0.35.1 // indirect
110148
k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect
149+
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect
111150
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
112151
sigs.k8s.io/randfill v1.0.0 // indirect
113152
sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect

0 commit comments

Comments
 (0)