Skip to content

Commit a52ce6d

Browse files
kvapsclaude
andcommitted
feat(grpc): split into Controller + Satellite services; satellite hosts gRPC server
The original single Satellite service had Hello (satellite→controller) mixed with ApplyResources/etc (controller→satellite), which forced both ends to be both client AND server through a single proto. Split into: - service Controller — Hello + ReportObserved (satellite-streaming observed events). Satellite is the client. - service Satellite — ApplyResources, ApplyStoragePools, snapshot RPCs, ShipSnapshot. Controller is the client. Agent.Run now optionally hosts the Satellite service on cfg.ListenAddr; cmd/satellite gains --listen, --lvm-pool-name, --lvm-vg, --lvm-thinpool flags so the agent can register an LVM-thin provider at startup. The Reconciler is reused — same code path that unit tests already cover. Co-Authored-By: Claude <noreply@anthropic.com> Signed-off-by: Andrei Kvapil <kvapss@gmail.com>
1 parent 1744f18 commit a52ce6d

11 files changed

Lines changed: 848 additions & 513 deletions

File tree

.golangci.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ linters:
214214
- cyclop
215215
- gocyclo
216216
- maintidx
217-
path: cmd/main.go
217+
path: cmd/(main|satellite/main)\.go
218218
# api/v1alpha1 and internal/controller are kubebuilder scaffold; the
219219
# boilerplate (dot-imports for ginkgo/gomega, generated DeepCopy globals,
220220
# ginkgo BeforeSuite TODOs, wrapcheck on framework returns) is not

cmd/satellite/main.go

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ import (
3434
"github.com/cockroachdb/errors"
3535

3636
"github.com/cozystack/blockstor/pkg/satellite"
37+
"github.com/cozystack/blockstor/pkg/storage"
38+
"github.com/cozystack/blockstor/pkg/storage/lvm"
3739
)
3840

3941
func main() {
@@ -48,6 +50,10 @@ func run() int {
4850
controllerAddr string
4951
nodeName string
5052
stateDir string
53+
listenAddr string
54+
lvmPoolName string
55+
lvmVG string
56+
lvmThinPool string
5157
)
5258

5359
flag.StringVar(&controllerAddr, "controller", "blockstor-controller:7000",
@@ -56,6 +62,14 @@ func run() int {
5662
"name this satellite registers under (defaults to NODE_NAME env)")
5763
flag.StringVar(&stateDir, "state-dir", "/var/lib/blockstor-satellite",
5864
"directory the satellite uses to persist DRBD .res files and per-resource state")
65+
flag.StringVar(&listenAddr, "listen", ":7000",
66+
"bind address for the satellite-side gRPC server (controller dials this for ApplyResources)")
67+
flag.StringVar(&lvmPoolName, "lvm-pool-name", "",
68+
"register an LVM-thin pool under this LINSTOR pool name (empty disables LVM)")
69+
flag.StringVar(&lvmVG, "lvm-vg", "",
70+
"LVM volume group backing the lvm-pool-name pool")
71+
flag.StringVar(&lvmThinPool, "lvm-thinpool", "",
72+
"LVM thinpool LV backing the lvm-pool-name pool")
5973
flag.Parse()
6074

6175
logger := slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))
@@ -70,18 +84,41 @@ func run() int {
7084
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
7185
defer cancel()
7286

87+
providers := map[string]storage.Provider{}
88+
89+
if lvmPoolName != "" {
90+
if lvmVG == "" || lvmThinPool == "" {
91+
logger.Error("lvm-pool-name set but lvm-vg / lvm-thinpool missing")
92+
93+
return 1
94+
}
95+
96+
providers[lvmPoolName] = lvm.NewThin(
97+
lvm.ThinConfig{VolumeGroup: lvmVG, ThinPool: lvmThinPool},
98+
storage.RealExec{})
99+
}
100+
73101
agent := satellite.NewAgent(satellite.Config{
74102
NodeName: nodeName,
75103
ControllerAddr: controllerAddr,
104+
ListenAddr: listenAddr,
76105
StateDir: stateDir,
106+
Providers: providers,
77107
DialTimeout: 10 * time.Second,
78108
Logger: logger,
79109
})
80110

111+
providerNames := make([]string, 0, len(providers))
112+
for name := range providers {
113+
providerNames = append(providerNames, name)
114+
}
115+
81116
logger.Info("blockstor-satellite starting",
82117
"node_name", nodeName,
83118
"controller", controllerAddr,
84-
"state_dir", stateDir)
119+
"state_dir", stateDir,
120+
"listen", listenAddr,
121+
"providers", providerNames)
85122

86123
err := agent.Run(ctx)
87124
if err != nil && !errors.Is(err, context.Canceled) {

pkg/satellite/agent.go

Lines changed: 78 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,16 @@ package satellite
2626
import (
2727
"context"
2828
"log/slog"
29+
"net"
2930
"time"
3031

3132
"github.com/cockroachdb/errors"
3233
"google.golang.org/grpc"
3334
"google.golang.org/grpc/credentials/insecure"
3435

36+
"github.com/cozystack/blockstor/pkg/drbd"
3537
satellitepb "github.com/cozystack/blockstor/pkg/satellite/proto"
38+
"github.com/cozystack/blockstor/pkg/storage"
3639
"github.com/cozystack/blockstor/pkg/version"
3740
)
3841

@@ -45,10 +48,21 @@ type Config struct {
4548
// ControllerAddr is the gRPC dial address of the blockstor-controller.
4649
ControllerAddr string
4750

51+
// ListenAddr is the bind address for the satellite's own gRPC server
52+
// (the side that the controller dials for ApplyResources, snapshot
53+
// RPCs, ship). Empty disables the server (useful for unit tests).
54+
ListenAddr string
55+
4856
// StateDir is the on-disk directory the satellite uses for DRBD .res
4957
// files and per-resource state. Required.
5058
StateDir string
5159

60+
// Providers maps storage-pool name → provider implementation. The
61+
// satellite reconciler uses this to resolve which backend a
62+
// DesiredVolume's StoragePool refers to. Seeded at startup from CLI
63+
// flags / env (LVM-thin, ZFS, file).
64+
Providers map[string]storage.Provider
65+
5266
// DialTimeout caps gRPC connection establishment per attempt.
5367
DialTimeout time.Duration
5468

@@ -65,6 +79,8 @@ type Agent struct {
6579
}
6680

6781
// NewAgent constructs an Agent without yet dialling the controller.
82+
//
83+
//nolint:gocritic // value receiver is the ergonomic public API; Config is the binary's flag bundle.
6884
func NewAgent(cfg Config) *Agent {
6985
logger := cfg.Logger
7086
if logger == nil {
@@ -75,11 +91,9 @@ func NewAgent(cfg Config) *Agent {
7591
}
7692

7793
// Run is the agent's main loop. It dials the controller, performs the
78-
// hello handshake to register the node, then keeps long-running apply
79-
// and observe loops supervised. When ctx is cancelled the loops drain
80-
// and Run returns ctx.Err().
81-
//
82-
// Phase 3.1: hello is the only real RPC; reconcile loops still no-op.
94+
// hello handshake to register the node, starts the satellite-side
95+
// gRPC server (so the controller can push desired state), then waits
96+
// for ctx to cancel.
8397
func (a *Agent) Run(ctx context.Context) error {
8498
if a.cfg.NodeName == "" {
8599
return errors.New("NodeName is required")
@@ -88,28 +102,84 @@ func (a *Agent) Run(ctx context.Context) error {
88102
a.logger.Info("agent starting",
89103
"node", a.cfg.NodeName,
90104
"blockstor_version", version.Version,
91-
"controller", a.cfg.ControllerAddr)
105+
"controller", a.cfg.ControllerAddr,
106+
"listen", a.cfg.ListenAddr)
92107

93108
conn, err := a.dial(ctx)
94109
if err != nil {
95110
return errors.Wrap(err, "dial controller")
96111
}
97112
defer func() { _ = conn.Close() }()
98113

99-
client := satellitepb.NewSatelliteClient(conn)
114+
client := satellitepb.NewControllerClient(conn)
100115

101116
err = a.hello(ctx, client)
102117
if err != nil {
103118
return errors.Wrap(err, "hello")
104119
}
105120

121+
// Bring up the satellite-side gRPC server so the controller can push
122+
// ApplyResources / snapshot RPCs at us. The Reconciler is wired with
123+
// the configured providers + drbdadm wrapper + state dir.
124+
srv, stop, err := a.startGRPCServer(ctx)
125+
if err != nil {
126+
return errors.Wrap(err, "start gRPC server")
127+
}
128+
defer stop()
129+
130+
a.logger.Info("satellite gRPC ready", "addr", srv)
131+
106132
<-ctx.Done()
107133

108134
a.logger.Info("agent stopping", "node", a.cfg.NodeName)
109135

110136
return ctx.Err() //nolint:wrapcheck // bubbling ctx.Err() unwrapped is the convention
111137
}
112138

139+
// startGRPCServer binds the satellite's `service Satellite` listener.
140+
// Empty cfg.ListenAddr disables the server (returns a no-op stop) so
141+
// unit tests that only exercise Hello don't need a free port.
142+
func (a *Agent) startGRPCServer(ctx context.Context) (string, func(), error) {
143+
if a.cfg.ListenAddr == "" {
144+
return "<disabled>", func() {}, nil
145+
}
146+
147+
rec := NewReconciler(ReconcilerConfig{
148+
Providers: a.cfg.Providers,
149+
Adm: drbd.NewAdm(storage.RealExec{}),
150+
StateDir: a.cfg.StateDir,
151+
NodeName: a.cfg.NodeName,
152+
})
153+
154+
listenCfg := &net.ListenConfig{}
155+
156+
listener, err := listenCfg.Listen(ctx, "tcp", a.cfg.ListenAddr)
157+
if err != nil {
158+
return "", nil, errors.Wrapf(err, "listen %s", a.cfg.ListenAddr)
159+
}
160+
161+
gs := grpc.NewServer()
162+
satellitepb.RegisterSatelliteServer(gs, NewGRPCServer(rec))
163+
164+
done := make(chan struct{})
165+
166+
go func() {
167+
defer close(done)
168+
169+
err := gs.Serve(listener)
170+
if err != nil {
171+
a.logger.Error("gRPC Serve returned", "err", err)
172+
}
173+
}()
174+
175+
stop := func() {
176+
gs.GracefulStop()
177+
<-done
178+
}
179+
180+
return listener.Addr().String(), stop, nil
181+
}
182+
113183
// dial opens an insecure gRPC connection to the controller. TLS comes in
114184
// Phase 6 alongside the rest of the encryption work; cluster traffic is
115185
// expected to ride a private k8s network until then.
@@ -134,7 +204,7 @@ func (a *Agent) dial(ctx context.Context) (*grpc.ClientConn, error) {
134204
// hello is the registration handshake. The satellite tells the controller
135205
// who it is and what layers / providers it can drive; the controller
136206
// upserts the corresponding Node CRD and replies with the cluster id.
137-
func (a *Agent) hello(ctx context.Context, client satellitepb.SatelliteClient) error {
207+
func (a *Agent) hello(ctx context.Context, client satellitepb.ControllerClient) error {
138208
rpcCtx, cancel := context.WithTimeout(ctx, a.cfg.DialTimeout)
139209
defer cancel()
140210

pkg/satellite/agent_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ func startServer(t *testing.T, st store.Store, clusterID string) (string, func()
119119

120120
srv := satellitecontroller.New(st, satellitecontroller.Config{ClusterID: clusterID})
121121
gs := grpc.NewServer()
122-
satellitepb.RegisterSatelliteServer(gs, srv)
122+
satellitepb.RegisterControllerServer(gs, srv)
123123

124124
errCh := make(chan error, 1)
125125
go func() { errCh <- gs.Serve(ln) }()

pkg/satellite/grpc_server.go

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
Copyright 2026 Cozystack contributors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package satellite
18+
19+
import (
20+
"context"
21+
22+
satellitepb "github.com/cozystack/blockstor/pkg/satellite/proto"
23+
)
24+
25+
// GRPCServer is the satellite-side implementation of `service Satellite`.
26+
// It glues controller→satellite RPCs (ApplyResources, ApplyStoragePools,
27+
// CreateSnapshot, DeleteSnapshot, ShipSnapshot) onto an in-process
28+
// Reconciler. Agent.Run wires this into a gRPC listener so the
29+
// controller can push desired state.
30+
type GRPCServer struct {
31+
satellitepb.UnimplementedSatelliteServer
32+
33+
rec *Reconciler
34+
}
35+
36+
// NewGRPCServer constructs a server backed by rec.
37+
func NewGRPCServer(rec *Reconciler) *GRPCServer {
38+
return &GRPCServer{rec: rec}
39+
}
40+
41+
// ApplyResources delegates to Reconciler.Apply. Per-resource failures
42+
// are surfaced via ResourceApplyResult.Ok=false rather than as gRPC
43+
// errors so a single bad replica doesn't sink a batched apply.
44+
func (g *GRPCServer) ApplyResources(ctx context.Context, req *satellitepb.ApplyResourcesRequest) (*satellitepb.ApplyResourcesResponse, error) {
45+
results, err := g.rec.Apply(ctx, req.GetResources())
46+
if err != nil {
47+
return nil, err
48+
}
49+
50+
return &satellitepb.ApplyResourcesResponse{Results: results}, nil
51+
}
52+
53+
// ApplyStoragePools is a placeholder that just OKs every requested pool
54+
// for now. The satellite uses a Provider registry seeded at startup
55+
// (CLI flags / env), so the controller's pool spec is informational
56+
// today; we'll wire dynamic pool wiring once the per-pool runtime
57+
// state lands.
58+
func (g *GRPCServer) ApplyStoragePools(_ context.Context, req *satellitepb.ApplyStoragePoolsRequest) (*satellitepb.ApplyStoragePoolsResponse, error) {
59+
pools := req.GetPools()
60+
results := make([]*satellitepb.StoragePoolApplyResult, 0, len(pools))
61+
62+
for _, pool := range pools {
63+
results = append(results, &satellitepb.StoragePoolApplyResult{
64+
Name: pool.GetName(),
65+
Ok: true,
66+
})
67+
}
68+
69+
return &satellitepb.ApplyStoragePoolsResponse{Results: results}, nil
70+
}
71+
72+
// CreateSnapshot routes through the Reconciler's existing snapshot
73+
// path. Per-snapshot errors land in the response body (Ok=false), the
74+
// gRPC error path is reserved for context cancellation / transport
75+
// faults.
76+
func (g *GRPCServer) CreateSnapshot(ctx context.Context, req *satellitepb.CreateSnapshotRequest) (*satellitepb.CreateSnapshotResponse, error) {
77+
return g.rec.CreateSnapshot(ctx, req)
78+
}
79+
80+
// DeleteSnapshot mirrors CreateSnapshot.
81+
func (g *GRPCServer) DeleteSnapshot(ctx context.Context, req *satellitepb.DeleteSnapshotRequest) (*satellitepb.DeleteSnapshotResponse, error) {
82+
return g.rec.DeleteSnapshot(ctx, req)
83+
}
84+
85+
// ShipSnapshot routes through the Reconciler's ship dispatch.
86+
func (g *GRPCServer) ShipSnapshot(ctx context.Context, req *satellitepb.ShipSnapshotRequest) (*satellitepb.ShipSnapshotResponse, error) {
87+
return g.rec.ShipSnapshot(ctx, req)
88+
}

0 commit comments

Comments
 (0)