Skip to content

finalizer halted due to error: halting Sequencer because of error reprocessing full batch #28

Open
Open
finalizer halted due to error: halting Sequencer because of error reprocessing full batch#28

Description

@tony-armstrong
tony-armstrong
opened on Feb 2, 2024
  1. first, an error will be reported ResourceExhausted
  2. then it will halt
  3. error thrown location

    b2-zkevm-node/state/batch.go

    Lines 347 to 352 in d16f73d

    res, err := s.executorClient.ProcessBatch(ctx, processBatchRequest)
    if err != nil {
    log.Errorf("Error s.executorClient.ProcessBatch: %v", err)
    log.Errorf("Error s.executorClient.ProcessBatch: %s", err.Error())
    log.Errorf("Error s.executorClient.ProcessBatch response: %v", res)
    } else if res.Error != executor.ExecutorError_EXECUTOR_ERROR_NO_ERROR {
  4. log 
    2024-02-02T05:02:04.789471419+08:00 stderr F 2024-02-02T05:02:04.789+0800	DEBUG	state/batch.go:341	processBatch[processBatchRequest.UpdateMerkleTree]: 0	{"pid": 1, "version": ""}
2024-02-02T05:02:04.789489648+08:00 stderr F 2024-02-02T05:02:04.789+0800	DEBUG	state/batch.go:342	processBatch[processBatchRequest.ChainId]: 1102	{"pid": 1, "version": ""}
2024-02-02T05:02:04.789492901+08:00 stderr F 2024-02-02T05:02:04.789+0800	DEBUG	state/batch.go:343	processBatch[processBatchRequest.ForkId]: 6	{"pid": 1, "version": ""}
2024-02-02T05:02:04.789507316+08:00 stderr F 2024-02-02T05:02:04.789+0800	DEBUG	state/batch.go:344	processBatch[processBatchRequest.ContextId]: 9ad86617-3138-40f7-9cee-59c81669c77e	{"pid": 1, "version": ""}
2024-02-02T05:02:04.789938537+08:00 stderr F 2024-02-02T05:02:04.789+0800	ERROR	state/batch.go:349	Error s.executorClient.ProcessBatch: rpc error: code = ResourceExhausted desc = %!(EXTRA string=
2024-02-02T05:02:04.789950516+08:00 stderr F /src/log/log.go:142 github.com/0xPolygonHermez/zkevm-node/log.appendStackTraceMaybeArgs()
2024-02-02T05:02:04.789954175+08:00 stderr F /src/log/log.go:251 github.com/0xPolygonHermez/zkevm-node/log.Errorf()
2024-02-02T05:02:04.789957163+08:00 stderr F /src/state/batch.go:349 github.com/0xPolygonHermez/zkevm-node/state.(*State).sendBatchRequestToExecutor()
2024-02-02T05:02:04.789960125+08:00 stderr F /src/state/batch.go:188 github.com/0xPolygonHermez/zkevm-node/state.(*State).ProcessBatch()
2024-02-02T05:02:04.789963216+08:00 stderr F /src/sequencer/finalizer.go:1307 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).reprocessFullBatch()
2024-02-02T05:02:04.78996594+08:00 stderr F /src/sequencer/finalizer.go:509 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).newWIPBatch()
2024-02-02T05:02:04.789968792+08:00 stderr F /src/sequencer/finalizer.go:429 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatch()
2024-02-02T05:02:04.789971691+08:00 stderr F /src/sequencer/finalizer.go:381 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatches()
2024-02-02T05:02:04.789976777+08:00 stderr F /src/sequencer/finalizer.go:194 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).Start()
2024-02-02T05:02:04.789980204+08:00 stderr F )	{"pid": 1, "version": ""}
2024-02-02T05:02:04.789982781+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/state.(*State).sendBatchRequestToExecutor
2024-02-02T05:02:04.789985626+08:00 stderr F 	/src/state/batch.go:349
2024-02-02T05:02:04.789988564+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/state.(*State).ProcessBatch
2024-02-02T05:02:04.789991505+08:00 stderr F 	/src/state/batch.go:188
2024-02-02T05:02:04.789994431+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).reprocessFullBatch
2024-02-02T05:02:04.789997557+08:00 stderr F 	/src/sequencer/finalizer.go:1307
2024-02-02T05:02:04.79000018+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).newWIPBatch
2024-02-02T05:02:04.790003008+08:00 stderr F 	/src/sequencer/finalizer.go:509
2024-02-02T05:02:04.790006306+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatch
2024-02-02T05:02:04.790009233+08:00 stderr F 	/src/sequencer/finalizer.go:429
2024-02-02T05:02:04.790012305+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatches
2024-02-02T05:02:04.790014969+08:00 stderr F 	/src/sequencer/finalizer.go:381
2024-02-02T05:02:04.790017642+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).Start
2024-02-02T05:02:04.790020513+08:00 stderr F 	/src/sequencer/finalizer.go:194
2024-02-02T05:02:04.790023594+08:00 stderr F 2024-02-02T05:02:04.789+0800	ERROR	state/batch.go:350	Error s.executorClient.ProcessBatch: rpc error: code = ResourceExhausted desc = 	{"pid": 1, "version": ""}
2024-02-02T05:02:04.790026645+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/state.(*State).sendBatchRequestToExecutor
2024-02-02T05:02:04.79002939+08:00 stderr F 	/src/state/batch.go:350
2024-02-02T05:02:04.790032252+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/state.(*State).ProcessBatch
2024-02-02T05:02:04.790043219+08:00 stderr F 	/src/state/batch.go:188
2024-02-02T05:02:04.790045912+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).reprocessFullBatch
2024-02-02T05:02:04.790048792+08:00 stderr F 	/src/sequencer/finalizer.go:1307
2024-02-02T05:02:04.790051686+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).newWIPBatch
2024-02-02T05:02:04.790054652+08:00 stderr F 	/src/sequencer/finalizer.go:509
2024-02-02T05:02:04.790057531+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatch
2024-02-02T05:02:04.790059995+08:00 stderr F 	/src/sequencer/finalizer.go:429
2024-02-02T05:02:04.790063362+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatches
2024-02-02T05:02:04.790066423+08:00 stderr F 	/src/sequencer/finalizer.go:381
2024-02-02T05:02:04.790069179+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).Start
2024-02-02T05:02:04.790071997+08:00 stderr F 	/src/sequencer/finalizer.go:194
2024-02-02T05:02:04.790094396+08:00 stderr F 2024-02-02T05:02:04.789+0800	ERROR	state/batch.go:351	Error s.executorClient.ProcessBatch response: <nil>	{"pid": 1, "version": ""}
2024-02-02T05:02:04.790114018+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/state.(*State).sendBatchRequestToExecutor
2024-02-02T05:02:04.790116925+08:00 stderr F 	/src/state/batch.go:351
2024-02-02T05:02:04.790119505+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/state.(*State).ProcessBatch
2024-02-02T05:02:04.790122378+08:00 stderr F 	/src/state/batch.go:188
2024-02-02T05:02:04.79012516+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).reprocessFullBatch
2024-02-02T05:02:04.790127819+08:00 stderr F 	/src/sequencer/finalizer.go:1307
2024-02-02T05:02:04.790130337+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).newWIPBatch
2024-02-02T05:02:04.790132934+08:00 stderr F 	/src/sequencer/finalizer.go:509
2024-02-02T05:02:04.790135863+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatch
2024-02-02T05:02:04.790138604+08:00 stderr F 	/src/sequencer/finalizer.go:429
2024-02-02T05:02:04.790141517+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatches
2024-02-02T05:02:04.790144785+08:00 stderr F 	/src/sequencer/finalizer.go:381
2024-02-02T05:02:04.790147591+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).Start
2024-02-02T05:02:04.790150575+08:00 stderr F 	/src/sequencer/finalizer.go:194
2024-02-02T05:02:04.790154118+08:00 stderr F 2024-02-02T05:02:04.789+0800	INFO	state/batch.go:360	Batch: 27496 took 513.895µs to be processed by the executor 	{"pid": 1, "version": ""}
2024-02-02T05:02:04.790164874+08:00 stderr F 2024-02-02T05:02:04.789+0800	ERROR	sequencer/finalizer.go:1309	reprocessFullBatch: failed to process batch 27496. Error: rpc error: code = ResourceExhausted desc = %!(EXTRA string=
2024-02-02T05:02:04.790172721+08:00 stderr F /src/log/log.go:142 github.com/0xPolygonHermez/zkevm-node/log.appendStackTraceMaybeArgs()
2024-02-02T05:02:04.790175392+08:00 stderr F /src/log/log.go:251 github.com/0xPolygonHermez/zkevm-node/log.Errorf()
2024-02-02T05:02:04.790177917+08:00 stderr F /src/sequencer/finalizer.go:1309 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).reprocessFullBatch()
2024-02-02T05:02:04.790180828+08:00 stderr F /src/sequencer/finalizer.go:509 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).newWIPBatch()
2024-02-02T05:02:04.790186618+08:00 stderr F /src/sequencer/finalizer.go:429 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatch()
2024-02-02T05:02:04.790191599+08:00 stderr F /src/sequencer/finalizer.go:381 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatches()
2024-02-02T05:02:04.790194234+08:00 stderr F /src/sequencer/finalizer.go:194 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).Start()
2024-02-02T05:02:04.790203078+08:00 stderr F )	{"pid": 1, "version": ""}
2024-02-02T05:02:04.790205838+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).reprocessFullBatch
2024-02-02T05:02:04.790208835+08:00 stderr F 	/src/sequencer/finalizer.go:1309
2024-02-02T05:02:04.79021183+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).newWIPBatch
2024-02-02T05:02:04.790214802+08:00 stderr F 	/src/sequencer/finalizer.go:509
2024-02-02T05:02:04.790217442+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatch
2024-02-02T05:02:04.790221166+08:00 stderr F 	/src/sequencer/finalizer.go:429
2024-02-02T05:02:04.790224174+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatches
2024-02-02T05:02:04.790226894+08:00 stderr F 	/src/sequencer/finalizer.go:381
2024-02-02T05:02:04.790229398+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).Start
2024-02-02T05:02:04.790232078+08:00 stderr F 	/src/sequencer/finalizer.go:194
2024-02-02T05:02:04.790235038+08:00 stderr F 2024-02-02T05:02:04.790+0800	ERROR	nileventstorage/nileventstorage.go:30	Event: &{Id:{neg:false abs:[]} ReceivedAt:2024-02-02 05:02:04.790001958 +0800 CST m=+582.609952599 IPAddress: Source:node Component:sequencer Level:crit EventID:FINALIZER HALT Description:finalizer halted due to error: halting Sequencer because of error reprocessing full batch 27496 (sanity check). Error: processing batch error  Data:[] Json:<nil>}	{"pid": 1, "version": ""}
2024-02-02T05:02:04.790237709+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/event/nileventstorage.LogEvent
2024-02-02T05:02:04.790240918+08:00 stderr F 	/src/event/nileventstorage/nileventstorage.go:30
2024-02-02T05:02:04.790243568+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/event/pgeventstorage.(*PostgresEventStorage).LogEvent
2024-02-02T05:02:04.790246133+08:00 stderr F 	/src/event/pgeventstorage/pgeventstorage.go:45
2024-02-02T05:02:04.790248711+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/event.(*EventLog).LogEvent
2024-02-02T05:02:04.790251573+08:00 stderr F 	/src/event/eventlog.go:28
2024-02-02T05:02:04.790254156+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).halt
2024-02-02T05:02:04.790256764+08:00 stderr F 	/src/sequencer/finalizer.go:447
2024-02-02T05:02:04.790259587+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).newWIPBatch
2024-02-02T05:02:04.790262506+08:00 stderr F 	/src/sequencer/finalizer.go:512
2024-02-02T05:02:04.790265325+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatch
2024-02-02T05:02:04.790267831+08:00 stderr F 	/src/sequencer/finalizer.go:429
2024-02-02T05:02:04.790270275+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatches
2024-02-02T05:02:04.790272836+08:00 stderr F 	/src/sequencer/finalizer.go:381
2024-02-02T05:02:04.790275403+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).Start
2024-02-02T05:02:04.790277963+08:00 stderr F 	/src/sequencer/finalizer.go:194
2024-02-02T05:02:04.790857566+08:00 stderr F 2024-02-02T05:02:04.790+0800	ERROR	sequencer/finalizer.go:453	fatal error: halting Sequencer because of error reprocessing full batch 27496 (sanity check). Error: processing batch error %!(EXTRA string=
2024-02-02T05:02:04.790871568+08:00 stderr F /src/log/log.go:142 github.com/0xPolygonHermez/zkevm-node/log.appendStackTraceMaybeArgs()
2024-02-02T05:02:04.790874914+08:00 stderr F /src/log/log.go:251 github.com/0xPolygonHermez/zkevm-node/log.Errorf()
2024-02-02T05:02:04.790877885+08:00 stderr F /src/sequencer/finalizer.go:453 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).halt()
2024-02-02T05:02:04.790880894+08:00 stderr F /src/sequencer/finalizer.go:512 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).newWIPBatch()
2024-02-02T05:02:04.790883826+08:00 stderr F /src/sequencer/finalizer.go:429 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatch()
2024-02-02T05:02:04.790893071+08:00 stderr F /src/sequencer/finalizer.go:381 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatches()
2024-02-02T05:02:04.790895921+08:00 stderr F /src/sequencer/finalizer.go:194 github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).Start()
2024-02-02T05:02:04.790899187+08:00 stderr F )	{"pid": 1, "version": ""}
2024-02-02T05:02:04.790902091+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).halt
2024-02-02T05:02:04.790904649+08:00 stderr F 	/src/sequencer/finalizer.go:453
2024-02-02T05:02:04.790907461+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).newWIPBatch
2024-02-02T05:02:04.790910555+08:00 stderr F 	/src/sequencer/finalizer.go:512
2024-02-02T05:02:04.790913558+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatch
2024-02-02T05:02:04.790916331+08:00 stderr F 	/src/sequencer/finalizer.go:429
2024-02-02T05:02:04.790920908+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatches
2024-02-02T05:02:04.790924285+08:00 stderr F 	/src/sequencer/finalizer.go:381
2024-02-02T05:02:04.790927359+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).Start
2024-02-02T05:02:04.790930111+08:00 stderr F 	/src/sequencer/finalizer.go:194
2024-02-02T05:02:04.790932998+08:00 stderr F 2024-02-02T05:02:04.790+0800	ERROR	sequencer/finalizer.go:454	halting the finalizer	{"pid": 1, "version": ""}
2024-02-02T05:02:04.790935987+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).halt
2024-02-02T05:02:04.790938792+08:00 stderr F 	/src/sequencer/finalizer.go:454
2024-02-02T05:02:04.790941707+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).newWIPBatch
2024-02-02T05:02:04.790944528+08:00 stderr F 	/src/sequencer/finalizer.go:512
2024-02-02T05:02:04.790947739+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatch
2024-02-02T05:02:04.790951298+08:00 stderr F 	/src/sequencer/finalizer.go:429
2024-02-02T05:02:04.790954298+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).finalizeBatches
2024-02-02T05:02:04.790957181+08:00 stderr F 	/src/sequencer/finalizer.go:381
2024-02-02T05:02:04.79096006+08:00 stderr F github.com/0xPolygonHermez/zkevm-node/sequencer.(*finalizer).Start
2024-02-02T05:02:04.790963435+08:00 stderr F 	/src/sequencer/finalizer.go:194
2024-02-02T05:02:04.965183501+08:00 stderr F 2024-02-02T05:02:04.965+0800	INFO	sequencer/worker.go:95	AddTx new tx(0x4af0bb3bf9ca5b1775cf9b3ef0f53cf6619d3eb590548ce07937cbe195fcd8bd) nonce(551171) gasPrice(10000000) to addrQueue(0x35fE57b69964227E67795288746508A7F89b287d) nonce(551171) balance(448713248671840000000)	{"pid": 1, "version": ""}
2024-02-02T05:02:04.965199751+08:00 stderr F 2024-02-02T05:02:04.965+0800	INFO	sequencer/worker.go:110	AddTx newReadyTx(0x4af0bb3bf9ca5b1775cf9b3ef0f53cf6619d3eb590548ce07937cbe195fcd8bd) nonce(551171) gasPrice(10000000) addr(0x35fE57b69964227E67795288746508A7F89b287d) added to TxSortedList	{"pid": 1, "version": ""}

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Fields

    No fields configured for issues without a type.

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions