Skip to content

Commit a1d058c

Browse files
gupadhyayaGanesha UpadhyayaManav-Aggarwalnashqueue
authored
Support multi sequencer (#823)
fixes #819 and #816 --------- Co-authored-by: Ganesha Upadhyaya <gupadhyaya@Ganeshas-MacBook-Pro-2.local> Co-authored-by: Manav Aggarwal <manavaggarwal1234@gmail.com> Co-authored-by: nashqueue <99758629+nashqueue@users.noreply.github.com>
1 parent 835a470 commit a1d058c

3 files changed

Lines changed: 128 additions & 51 deletions

File tree

‎block/manager.go‎

Lines changed: 48 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package block
22

33
import (
4+
"bytes"
45
"context"
6+
"encoding/hex"
57
"fmt"
68
"sync"
79
"sync/atomic"
@@ -68,6 +70,8 @@ type Manager struct {
6870
// retrieveCond is used to notify sync goroutine (SyncLoop) that it needs to retrieve data
6971
retrieveCond *sync.Cond
7072

73+
lastStateMtx *sync.Mutex
74+
7175
logger log.Logger
7276

7377
// For usage by Lazy Aggregator mode
@@ -151,6 +155,7 @@ func NewManager(
151155
blockInCh: make(chan newBlockEvent, 100),
152156
FraudProofInCh: make(chan *abci.FraudProof, 100),
153157
retrieveMtx: new(sync.Mutex),
158+
lastStateMtx: new(sync.Mutex),
154159
syncCache: make(map[uint64]*types.Block),
155160
logger: logger,
156161
txsAvailable: txsAvailableCh,
@@ -340,10 +345,18 @@ func (m *Manager) trySyncNextBlock(ctx context.Context, daHeight uint64) error {
340345
return fmt.Errorf("failed to save block responses: %w", err)
341346
}
342347

348+
// SaveValidators commits the DB tx
349+
err = m.store.SaveValidators(uint64(b.SignedHeader.Header.Height()), m.lastState.Validators)
350+
if err != nil {
351+
return err
352+
}
353+
343354
if daHeight > newState.DAHeight {
344355
newState.DAHeight = daHeight
345356
}
357+
m.lastStateMtx.Lock()
346358
m.lastState = newState
359+
m.lastStateMtx.Unlock()
347360
err = m.store.UpdateState(m.lastState)
348361
if err != nil {
349362
m.logger.Error("failed to save updated state", "error", err)
@@ -449,13 +462,37 @@ func (m *Manager) getCommit(header types.Header) (*types.Commit, error) {
449462
}, nil
450463
}
451464

465+
func (m *Manager) IsProposer() (bool, error) {
466+
// if proposer is not set, assume self proposer
467+
if m.lastState.Validators.Proposer == nil {
468+
return true, nil
469+
}
470+
471+
signerPubBytes, err := m.proposerKey.GetPublic().Raw()
472+
if err != nil {
473+
return false, err
474+
}
475+
476+
return bytes.Equal(m.lastState.Validators.Proposer.PubKey.Bytes(), signerPubBytes), nil
477+
}
478+
452479
func (m *Manager) publishBlock(ctx context.Context) error {
453480
var lastCommit *types.Commit
454481
var lastHeaderHash types.Hash
455482
var err error
456483
height := m.store.Height()
457484
newHeight := height + 1
458485

486+
m.lastStateMtx.Lock()
487+
isProposer, err := m.IsProposer()
488+
m.lastStateMtx.Unlock()
489+
if err != nil {
490+
return fmt.Errorf("error while checking for proposer: %w", err)
491+
}
492+
if !isProposer {
493+
return nil
494+
}
495+
459496
// this is a special case, when first block is produced - there is no previous commit
460497
if newHeight == uint64(m.genesis.InitialHeight) {
461498
lastCommit = &types.Commit{}
@@ -527,6 +564,9 @@ func (m *Manager) publishBlock(ctx context.Context) error {
527564
return err
528565
}
529566

567+
// Only update the stored height after successfully submitting to DA layer and committing to the DB
568+
m.store.SetHeight(uint64(block.SignedHeader.Header.Height()))
569+
530570
// Commit the new state and block which writes to disk on the proxy app
531571
_, _, err = m.executor.Commit(ctx, newState, block, responses)
532572
if err != nil {
@@ -539,6 +579,12 @@ func (m *Manager) publishBlock(ctx context.Context) error {
539579
return err
540580
}
541581

582+
// SaveValidators commits the DB tx
583+
err = m.store.SaveValidators(uint64(block.SignedHeader.Header.Height()), m.lastState.Validators)
584+
if err != nil {
585+
return err
586+
}
587+
542588
newState.DAHeight = atomic.LoadUint64(&m.daHeight)
543589
// After this call m.lastState is the NEW state returned from ApplyBlock
544590
m.lastState = newState
@@ -549,18 +595,11 @@ func (m *Manager) publishBlock(ctx context.Context) error {
549595
return err
550596
}
551597

552-
// SaveValidators commits the DB tx
553-
err = m.store.SaveValidators(uint64(block.SignedHeader.Header.Height()), m.lastState.Validators)
554-
if err != nil {
555-
return err
556-
}
557-
558-
// Only update the stored height after successfully submitting to DA layer and committing to the DB
559-
m.store.SetHeight(uint64(block.SignedHeader.Header.Height()))
560-
561598
// Publish header to channel so that header exchange service can broadcast
562599
m.HeaderCh <- &block.SignedHeader
563600

601+
m.logger.Debug("successfully proposed block", "proposer", hex.EncodeToString(block.SignedHeader.ProposerAddress), "height", block.SignedHeader.Height())
602+
564603
return nil
565604
}
566605

‎node/full_client_test.go‎

Lines changed: 79 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@ import (
3131
"github.com/rollkit/rollkit/config"
3232
"github.com/rollkit/rollkit/conv"
3333
abciconv "github.com/rollkit/rollkit/conv/abci"
34+
mockda "github.com/rollkit/rollkit/da/mock"
3435
"github.com/rollkit/rollkit/mocks"
36+
"github.com/rollkit/rollkit/store"
3537
"github.com/rollkit/rollkit/types"
3638
)
3739

@@ -648,60 +650,73 @@ func TestBlockchainInfo(t *testing.T) {
648650
}
649651

650652
func TestValidatorSetHandling(t *testing.T) {
651-
// handle multiple sequencers
652-
t.Skip()
653-
654653
assert := assert.New(t)
655654
require := require.New(t)
656-
app := &mocks.Application{}
657-
app.On("InitChain", mock.Anything).Return(abci.ResponseInitChain{})
658-
app.On("CheckTx", mock.Anything).Return(abci.ResponseCheckTx{})
659-
app.On("BeginBlock", mock.Anything).Return(abci.ResponseBeginBlock{})
660-
app.On("Commit", mock.Anything).Return(abci.ResponseCommit{})
661-
app.On("GetAppHash", mock.Anything).Return(abci.ResponseGetAppHash{})
662-
app.On("GenerateFraudProof", mock.Anything).Return(abci.ResponseGenerateFraudProof{})
663655

664-
key, _, _ := crypto.GenerateEd25519Key(crand.Reader)
656+
waitCh := make(chan interface{})
657+
658+
vKeys := make([]tmcrypto.PrivKey, 2)
659+
apps := make([]*mocks.Application, 2)
660+
nodes := make([]*FullNode, 2)
665661

666-
vKeys := make([]tmcrypto.PrivKey, 4)
667662
genesisValidators := make([]tmtypes.GenesisValidator, len(vKeys))
668663
for i := 0; i < len(vKeys); i++ {
669664
vKeys[i] = ed25519.GenPrivKey()
670665
genesisValidators[i] = tmtypes.GenesisValidator{Address: vKeys[i].PubKey().Address(), PubKey: vKeys[i].PubKey(), Power: int64(i + 100), Name: fmt.Sprintf("gen #%d", i)}
666+
apps[i] = createApp(vKeys[0], waitCh, require)
671667
}
672668

673-
nodeKey := &p2p.NodeKey{
674-
PrivKey: vKeys[0],
675-
}
676-
signingKey, _ := conv.GetNodeKey(nodeKey)
677-
678-
pbValKey, err := encoding.PubKeyToProto(vKeys[0].PubKey())
679-
require.NoError(err)
680-
681-
waitCh := make(chan interface{})
682-
683-
app.On("EndBlock", mock.Anything).Return(abci.ResponseEndBlock{}).Times(5)
684-
app.On("EndBlock", mock.Anything).Return(abci.ResponseEndBlock{ValidatorUpdates: []abci.ValidatorUpdate{{PubKey: pbValKey, Power: 0}}}).Once()
685-
app.On("EndBlock", mock.Anything).Return(abci.ResponseEndBlock{}).Once()
686-
app.On("EndBlock", mock.Anything).Return(abci.ResponseEndBlock{ValidatorUpdates: []abci.ValidatorUpdate{{PubKey: pbValKey, Power: 100}}}).Once()
687-
app.On("EndBlock", mock.Anything).Return(abci.ResponseEndBlock{}).Run(func(args mock.Arguments) {
688-
waitCh <- nil
689-
})
669+
dalc := &mockda.DataAvailabilityLayerClient{}
670+
ds, err := store.NewDefaultInMemoryKVStore()
671+
require.Nil(err)
672+
err = dalc.Init([8]byte{}, nil, ds, log.TestingLogger())
673+
require.Nil(err)
674+
err = dalc.Start()
675+
require.Nil(err)
676+
677+
for i := 0; i < len(nodes); i++ {
678+
nodeKey := &p2p.NodeKey{
679+
PrivKey: vKeys[i],
680+
}
681+
signingKey, err := conv.GetNodeKey(nodeKey)
682+
require.NoError(err)
683+
nodes[i], err = newFullNode(
684+
context.Background(),
685+
config.NodeConfig{
686+
DALayer: "mock",
687+
Aggregator: true,
688+
BlockManagerConfig: config.BlockManagerConfig{
689+
BlockTime: 1 * time.Second,
690+
DABlockTime: 100 * time.Millisecond,
691+
},
692+
},
693+
signingKey,
694+
signingKey,
695+
abcicli.NewLocalClient(nil, apps[i]),
696+
&tmtypes.GenesisDoc{ChainID: "test", Validators: genesisValidators},
697+
log.TestingLogger(),
698+
)
699+
require.NoError(err)
700+
require.NotNil(nodes[i])
690701

691-
node, err := newFullNode(context.Background(), config.NodeConfig{DALayer: "mock", Aggregator: true, BlockManagerConfig: config.BlockManagerConfig{BlockTime: 10 * time.Millisecond}}, key, signingKey, abcicli.NewLocalClient(nil, app), &tmtypes.GenesisDoc{ChainID: "test", Validators: genesisValidators}, log.TestingLogger())
692-
require.NoError(err)
693-
require.NotNil(node)
702+
// use same, common DALC, so nodes can share data
703+
nodes[i].dalc = dalc
704+
nodes[i].blockManager.SetDALC(dalc)
705+
}
694706

695-
rpc := NewFullClient(node)
707+
rpc := NewFullClient(nodes[0])
696708
require.NotNil(rpc)
697709

698-
err = node.Start()
699-
require.NoError(err)
710+
for i := 0; i < len(nodes); i++ {
711+
err := nodes[i].Start()
712+
require.NoError(err)
713+
}
700714

715+
<-waitCh
701716
<-waitCh
702717

703718
// test first blocks
704-
for h := int64(1); h <= 6; h++ {
719+
for h := int64(1); h <= 3; h++ {
705720
vals, err := rpc.Validators(context.Background(), &h, nil, nil)
706721
assert.NoError(err)
707722
assert.NotNil(vals)
@@ -710,8 +725,8 @@ func TestValidatorSetHandling(t *testing.T) {
710725
assert.EqualValues(vals.BlockHeight, h)
711726
}
712727

713-
// 6th EndBlock removes first validator from the list
714-
for h := int64(7); h <= 8; h++ {
728+
// 3rd EndBlock removes the first validator from the list
729+
for h := int64(4); h <= 5; h++ {
715730
vals, err := rpc.Validators(context.Background(), &h, nil, nil)
716731
assert.NoError(err)
717732
assert.NotNil(vals)
@@ -720,8 +735,9 @@ func TestValidatorSetHandling(t *testing.T) {
720735
assert.EqualValues(vals.BlockHeight, h)
721736
}
722737

723-
// 8th EndBlock adds validator back
724-
for h := int64(9); h <= 12; h++ {
738+
// 5th EndBlock adds validator back
739+
for h := int64(6); h <= 9; h++ {
740+
<-waitCh
725741
<-waitCh
726742
vals, err := rpc.Validators(context.Background(), &h, nil, nil)
727743
assert.NoError(err)
@@ -737,7 +753,29 @@ func TestValidatorSetHandling(t *testing.T) {
737753
assert.NotNil(vals)
738754
assert.EqualValues(len(genesisValidators), vals.Total)
739755
assert.Len(vals.Validators, len(genesisValidators))
740-
assert.GreaterOrEqual(vals.BlockHeight, int64(12))
756+
assert.GreaterOrEqual(vals.BlockHeight, int64(9))
757+
}
758+
759+
func createApp(keyToRemove tmcrypto.PrivKey, waitCh chan interface{}, require *require.Assertions) *mocks.Application {
760+
app := &mocks.Application{}
761+
app.On("InitChain", mock.Anything).Return(abci.ResponseInitChain{})
762+
app.On("CheckTx", mock.Anything).Return(abci.ResponseCheckTx{})
763+
app.On("BeginBlock", mock.Anything).Return(abci.ResponseBeginBlock{})
764+
app.On("Commit", mock.Anything).Return(abci.ResponseCommit{})
765+
app.On("GetAppHash", mock.Anything).Return(abci.ResponseGetAppHash{})
766+
app.On("GenerateFraudProof", mock.Anything).Return(abci.ResponseGenerateFraudProof{})
767+
768+
pbValKey, err := encoding.PubKeyToProto(keyToRemove.PubKey())
769+
require.NoError(err)
770+
771+
app.On("EndBlock", mock.Anything).Return(abci.ResponseEndBlock{}).Times(2)
772+
app.On("EndBlock", mock.Anything).Return(abci.ResponseEndBlock{ValidatorUpdates: []abci.ValidatorUpdate{{PubKey: pbValKey, Power: 0}}}).Once()
773+
app.On("EndBlock", mock.Anything).Return(abci.ResponseEndBlock{}).Once()
774+
app.On("EndBlock", mock.Anything).Return(abci.ResponseEndBlock{ValidatorUpdates: []abci.ValidatorUpdate{{PubKey: pbValKey, Power: 100}}}).Once()
775+
app.On("EndBlock", mock.Anything).Return(abci.ResponseEndBlock{}).Run(func(args mock.Arguments) {
776+
waitCh <- nil
777+
})
778+
return app
741779
}
742780

743781
// copy-pasted from store/store_test.go

‎state/executor.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ func (e *BlockExecutor) updateState(state types.State, block *types.Block, abciR
231231
// for now, we don't care about part set headers
232232
},
233233
NextValidators: nValSet,
234-
Validators: state.NextValidators.Copy(),
234+
Validators: nValSet,
235235
LastValidators: state.Validators.Copy(),
236236
LastHeightValidatorsChanged: lastHeightValSetChanged,
237237
ConsensusParams: state.ConsensusParams,

0 commit comments

Comments
 (0)