Merge pull request #6489 from oasisprotocol/martin/breaking/explicit-observer-mode

Config: Add explicit observer mode and storage kind

Author: martintomazic

.changelog/6488.cfg.0.md

@@ -0,0 +1 @@
+oasis-node/config: Add explicit observer mode

.changelog/6488.cfg.1.md

@@ -0,0 +1,9 @@
+go/config: Add explicit local storage config
+
+Explicit `config.Config.Consensus.LocalStorage` (default `true`) was added.
+When set to `false`, remote authenticated storage is used and verified
+by the node's light client.
+
+Mode `stateless-client` no longer exists. It should be instead configured by
+`config.Config.Mode: client` and
+`config.Config.Consensus.LocalStorage: false`.

go/config/config.go

@@ -31,42 +31,23 @@ const (
 	ModeValidator NodeMode = "validator"
 	// ModeCompute is the name of the compute node mode.
 	ModeCompute NodeMode = "compute"
+	// ModeObserver is the name of the observer node mode.
+	ModeObserver NodeMode = "observer"
 	// ModeKeyManager is the name of the key manager node mode.
 	ModeKeyManager NodeMode = "keymanager"
 	// ModeClient is the name of the client node mode.
 	ModeClient NodeMode = "client"
-	// ModeStatelessClient is the name of the stateless client node mode.
-	ModeStatelessClient NodeMode = "client-stateless"
 	// ModeSeed is the name of the seed node mode.
 	ModeSeed NodeMode = "seed"
 	// ModeArchive is the name of the archive node mode.
 	ModeArchive NodeMode = "archive"
 )
 
-// IsClientOnly returns true iff the mode is one that has the node running
-// as a client for all configured runtimes.
-func (m NodeMode) IsClientOnly() bool {
-	switch m {
-	case ModeClient, ModeStatelessClient:
-		return true
-	}
-	return false
-}
-
 // IsArchive returns true iff the mode is set to archive node mode.
 func (m NodeMode) IsArchive() bool {
 	return m == ModeArchive
 }
 
-// HasLocalStorage returns true iff the mode is one that has local storage.
-func (m NodeMode) HasLocalStorage() bool {
-	switch m {
-	case ModeClient, ModeCompute, ModeArchive:
-		return true
-	}
-	return false
-}
-
 // GlobalConfig holds the global configuration options.
 var GlobalConfig Config
 
@@ -97,15 +78,26 @@ func (c *Config) Validate() error {
 	switch c.Mode {
 	case ModeValidator:
 	case ModeCompute:
+	case ModeObserver:
 	case ModeKeyManager:
 	case ModeClient:
-	case ModeStatelessClient:
 	case ModeSeed:
 	case ModeArchive:
 	default:
 		return fmt.Errorf("unknown node mode: %s", c.Mode)
 	}
 
+	if !c.Consensus.LocalStorage {
+		if c.Consensus.Validator {
+			return fmt.Errorf("local storage not available in specified mode")
+		}
+		switch c.Mode {
+		case ModeClient, ModeObserver:
+		default:
+			return fmt.Errorf("local storage not available in specified mode")
+		}
+	}
+
 	if err = c.Common.Validate(); err != nil {
 		return fmt.Errorf("common: %w", err)
 	}

go/config/config_test.go

@@ -0,0 +1,39 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRemoteStorageValidation(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		mode      NodeMode
+		validator bool
+		mustErr   bool
+	}{
+		{name: "client", mode: ModeClient},
+		{name: "observer", mode: ModeObserver},
+		{name: "validator", mode: ModeValidator, mustErr: true},
+		{name: "compute", mode: ModeCompute, mustErr: true},
+		{name: "keymanager", mode: ModeKeyManager, mustErr: true},
+		{name: "seed", mode: ModeSeed, mustErr: true},
+		{name: "archive", mode: ModeArchive, mustErr: true},
+		{name: "consensus validator and observer", mode: ModeObserver, validator: true, mustErr: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := DefaultConfig()
+			cfg.Mode = tc.mode
+			cfg.Consensus.LocalStorage = false
+			cfg.Consensus.Validator = tc.validator
+
+			err := cfg.Validate()
+			if tc.mustErr {
+				require.ErrorContains(t, err, "local storage not available in specified mode")
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}

go/consensus/cometbft/cometbft.go

@@ -37,26 +37,27 @@ func New(
 		return nil, err
 	}
 
-	switch config.GlobalConfig.Mode {
-	case config.ModeArchive:
+	if config.GlobalConfig.Mode == config.ModeArchive {
 		node, err := createArchiveNode(ctx, dataDir, identity, genesis, doc, genesisDoc)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create archive node: %w", err)
 		}
 		return node, nil
-	case config.ModeStatelessClient:
+	}
+
+	if !config.GlobalConfig.Consensus.LocalStorage {
 		node, err := createStatelessNode(ctx, dataDir, identity, genesis, doc, genesisDoc, p2p)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create stateless node: %w", err)
 		}
 		return node, nil
-	default:
-		node, err := createFullNode(ctx, dataDir, identity, genesis, doc, genesisDoc, upgrader, p2p)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create full node: %w", err)
-		}
-		return node, nil
 	}
+
+	node, err := createFullNode(ctx, dataDir, identity, genesis, doc, genesisDoc, upgrader, p2p)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create full node: %w", err)
+	}
+	return node, nil
 }
 
 func createArchiveNode(

go/consensus/cometbft/config/config.go

@@ -39,6 +39,12 @@ type Config struct {
 	// Average amount of time to delay shutting down the node on upgrade.
 	UpgradeStopDelay time.Duration `yaml:"upgrade_stop_delay,omitempty"`
 
+	// LocalStorage specifies whether consensus state is stored locally.
+	//
+	// If false and the node has additional runtime storage configured,
+	// that runtime storage is also accessed remotely.
+	LocalStorage bool `yaml:"local_storage,omitempty"`
+
 	// ABCI state pruning configuration.
 	Prune PruneConfig `yaml:"prune,omitempty"`
 
@@ -233,6 +239,7 @@ func DefaultConfig() Config {
 			MaxFee: 10_000_000_000,
 		},
 		UpgradeStopDelay: time.Minute,
+		LocalStorage:     true,
 		Prune: PruneConfig{
 			Strategy: PruneStrategyNone,
 			NumKept:  250_000,

go/oasis-node/cmd/node/node.go

@@ -277,7 +277,11 @@ func (n *Node) initRuntimeWorkers(genesisDoc *genesisAPI.Document) error {
 	n.svcMgr.Register(n.BeaconWorker)
 
 	// Initialize the storage worker.
+	storageCfg := workerStorage.Config{
+		HasLocalStorage: config.GlobalConfig.Consensus.LocalStorage,
+	}
 	n.StorageWorker, err = workerStorage.New(
+		storageCfg,
 		n.grpcInternal,
 		n.CommonWorker,
 		n.RegistrationWorker,
@@ -475,6 +479,20 @@ func NewNode() (node *Node, err error) { // nolint: gocyclo
 	}
 	node.EntityID = entityID
 
+	requiresEntity := config.GlobalConfig.Consensus.Validator
+	switch config.GlobalConfig.Mode {
+	case config.ModeValidator, config.ModeCompute, config.ModeKeyManager, config.ModeObserver:
+		requiresEntity = true
+	}
+
+	if requiresEntity && entityID == nil {
+		return nil, fmt.Errorf("node has no entity configured but expects one")
+	}
+
+	if config.GlobalConfig.Mode == config.ModeClient && !config.GlobalConfig.Consensus.Validator && entityID != nil {
+		return nil, fmt.Errorf("client node must not have entity configured, try using observer mode instead")
+	}
+
 	// Load configured values for all registered crash points.
 	crash.LoadViperArgValues()
 

go/oasis-node/cmd/node/node_control.go

@@ -270,7 +270,7 @@ func (n *Node) getRuntimeStatus(ctx context.Context) (map[common.Namespace]contr
 		}
 
 		// Take storage into account for last retained round.
-		if config.GlobalConfig.Mode.HasLocalStorage() {
+		if config.GlobalConfig.Consensus.LocalStorage {
 			lsb, ok := rt.Storage().(storage.LocalBackend)
 			switch ok {
 			case false:

go/oasis-test-runner/oasis/oasis.go

@@ -303,12 +303,12 @@ func (n *Node) Start() error {
 		n.Config.Mode = config.ModeArchive
 	}
 
-	switch n.Config.Mode {
-	case config.ModeStatelessClient:
+	if !n.Config.Consensus.LocalStorage {
 		for _, worker := range n.net.computeWorkers {
 			n.Config.Consensus.Providers = append(n.Config.Consensus.Providers, "unix:"+worker.SocketPath())
 		}
-	case config.ModeSeed:
+	}
+	if n.Config.Mode == config.ModeSeed {
 		n.Config.P2P.Discovery.Bootstrap.AllowPrivateIPs = true
 	}
 

go/oasis-test-runner/oasis/observer.go

@@ -72,7 +72,7 @@ func (o *Observer) ModifyConfig() error {
 		o.Config.Registration.EntityID = string(entityID)
 	}
 
-	o.Config.Mode = config.ModeClient
+	o.Config.Mode = config.ModeObserver
 	o.Config.Runtime.Provisioner = o.runtimeProvisioner
 	o.Config.Runtime.SGX.Loader = o.net.cfg.RuntimeSGXLoaderBinary
 	o.Config.Runtime.AttestInterval = o.net.cfg.RuntimeAttestInterval