Merge pull request #149 from fystack/barrier-implementation-improve-reliability

Barrier implementation / improve reliability

Author: anhthii

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
     branches: ["*"]
 
 env:
-  GO_VERSION: "1.25.5"
+  GO_VERSION: "1.25.8"
 
 jobs:
   test:

.github/workflows/e2e-tests.yml

@@ -7,7 +7,7 @@ on:
     branches: [master]
 
 env:
-  GO_VERSION: "1.24"
+  GO_VERSION: "1.25.8"
   CGO_ENABLED: 0
   DOCKER_BUILDKIT: 1
   GO_BUILD_FLAGS: -trimpath -ldflags="-s -w"

cmd/mpcium/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"net"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -188,15 +189,28 @@ func runNode(ctx context.Context, c *cli.Command) error {
 	}
 
 	pubsub := messaging.NewNATSPubSub(natsConn)
+	maxConcurrentKeygen := viper.GetInt("max_concurrent_keygen")
+	if maxConcurrentKeygen == 0 {
+		maxConcurrentKeygen = eventconsumer.DefaultConcurrentKeygen
+	}
+	maxConcurrentSigning := viper.GetInt("max_concurrent_signing")
+	if maxConcurrentSigning == 0 {
+		maxConcurrentSigning = eventconsumer.DefaultConcurrentSigning
+	}
+
 	keygenBroker, err := messaging.NewJetStreamBroker(ctx, natsConn, event.KeygenBrokerStream, []string{
 		event.KeygenRequestTopic,
-	})
+	},
+		messaging.WithMaxAckPending(maxConcurrentKeygen),
+	)
 	if err != nil {
 		logger.Fatal("Failed to create keygen jetstream broker", err)
 	}
 	signingBroker, err := messaging.NewJetStreamBroker(ctx, natsConn, event.SigningPublisherStream, []string{
 		event.SigningRequestTopic,
-	})
+	},
+		messaging.WithMaxAckPending(maxConcurrentSigning),
+	)
 	if err != nil {
 		logger.Fatal("Failed to create signing jetstream broker", err)
 	}
@@ -582,8 +596,27 @@ func GetNATSConnection(environment string, appConfig *config.AppConfig) (*nats.C
 	opts := []nats.Option{
 		nats.MaxReconnects(-1), // retry forever
 		nats.ReconnectWait(2 * time.Second),
-		nats.DisconnectHandler(func(nc *nats.Conn) {
-			logger.Warn("Disconnected from NATS")
+		nats.ReconnectBufSize(16 * 1024 * 1024), // 16MB buffer during reconnect
+		// Use a custom dialer with TCP keepalive to prevent servers from killing idle connections
+		nats.Dialer(&net.Dialer{
+			Timeout:   5 * time.Second,
+			KeepAlive: 30 * time.Second, // TCP keepalive every 30s — counts as wire activity for AWS
+		}),
+		// Ping every 20s, fail after 3 missed pings (60s).
+		// This detects dead connections well before AWS NAT Gateway's
+		// 350s idle timeout kills the TCP connection silently.
+		nats.PingInterval(20 * time.Second),
+		nats.MaxPingsOutstanding(3),
+		// Enable TCP keepalive to prevent network gateway from killing idle connections.
+		// This sends TCP-level keepalive probes that count as wire activity,
+		// preventing NAT Gateway / NLB idle timeout eviction.
+		nats.CustomInboxPrefix("_INBOX_mpcium"),
+		nats.DisconnectErrHandler(func(nc *nats.Conn, err error) {
+			if err != nil {
+				logger.Warn("Disconnected from NATS", "error", err.Error())
+			} else {
+				logger.Warn("Disconnected from NATS")
+			}
 		}),
 		nats.ReconnectHandler(func(nc *nats.Conn) {
 			logger.Info("Reconnected to NATS", "url", nc.ConnectedUrl())

config.yaml.template

@@ -21,7 +21,6 @@ backup_period_seconds: 300 # 5 minutes
 backup_dir: backups
 max_concurrent_keygen: 2
 max_concurrent_signing: 10
-session_warm_up_delay_ms: 100
 healthcheck:
   enabled: false  # disabled by default, set to true for cloud deployment
   address: "0.0.0.0:8080"

e2e/base_test.go

@@ -264,6 +264,79 @@ func (s *E2ETestSuite) RegisterPeers(t *testing.T) {
 	t.Log("Peer listing completed")
 }
 
+// SeedPreParams loads pre-generated ECDSA pre-parameters from fixture files
+// and writes them into each node's BadgerDB. This avoids the expensive safe-prime
+// generation at node startup which can take minutes on CI runners.
+//
+// Each node gets its own unique pre-parameters. Sharing the same pre-parameters
+// across nodes causes tss-lib to reject duplicate h1j values during keygen round 2.
+func (s *E2ETestSuite) SeedPreParams(t *testing.T) {
+	t.Log("Seeding pre-generated pre-params into node databases...")
+
+	// Read the badger password from a node config
+	configPath := filepath.Join(s.testDir, "test_node0", "config.yaml")
+	configData, err := os.ReadFile(configPath)
+	require.NoError(t, err, "Failed to read node config for badger password")
+
+	var nodeConfig struct {
+		BadgerPassword string `yaml:"badger_password"`
+		DbPath         string `yaml:"db_path"`
+	}
+	require.NoError(t, yaml.Unmarshal(configData, &nodeConfig), "Failed to parse node config")
+
+	dbBasePath := nodeConfig.DbPath
+	if dbBasePath == "" {
+		dbBasePath = "./db"
+	}
+
+	for i := 0; i < numNodes; i++ {
+		nodeName := fmt.Sprintf("test_node%d", i)
+
+		// Load per-node fixture files
+		fixtureData := make([][]byte, 2)
+		for j := 0; j < 2; j++ {
+			path := filepath.Join(s.testDir, "fixtures", fmt.Sprintf("node%d_pre_params_%d.json", i, j))
+			data, err := os.ReadFile(path)
+			if err != nil {
+				t.Logf("Warning: could not read pre-params fixture %s: %v (nodes will generate their own)", path, err)
+				return
+			}
+			fixtureData[j] = data
+		}
+
+		nodeDir := filepath.Join(s.testDir, nodeName)
+		dbPath := filepath.Join(nodeDir, dbBasePath, nodeName)
+
+		// Ensure the DB directory exists
+		require.NoError(t, os.MkdirAll(dbPath, 0755), "Failed to create DB directory for %s", nodeName)
+
+		// Open BadgerDB with the same options the node uses
+		opts := badger.DefaultOptions(dbPath).
+			WithCompression(options.ZSTD).
+			WithEncryptionKey([]byte(nodeConfig.BadgerPassword)).
+			WithIndexCacheSize(16 << 20).
+			WithBlockCacheSize(32 << 20).
+			WithLogger(nil)
+
+		db, err := badger.Open(opts)
+		require.NoError(t, err, "Failed to open BadgerDB for %s", nodeName)
+
+		// Write pre-params
+		for j := 0; j < 2; j++ {
+			key := fmt.Sprintf("pre_params_%d", j)
+			err := db.Update(func(txn *badger.Txn) error {
+				return txn.Set([]byte(key), fixtureData[j])
+			})
+			require.NoError(t, err, "Failed to seed %s for %s", key, nodeName)
+		}
+
+		require.NoError(t, db.Close(), "Failed to close BadgerDB for %s", nodeName)
+		t.Logf("Seeded pre-params for %s", nodeName)
+	}
+
+	t.Log("Pre-params seeding complete")
+}
+
 func (s *E2ETestSuite) StartNodes(t *testing.T) {
 	t.Log("Starting MPC nodes...")
 

e2e/cmd/generate-preparams/main.go

@@ -0,0 +1,62 @@
+// generate-preparams pre-computes ECDSA pre-parameters and writes them as
+// JSON fixture files. Run once from the e2e directory (or whenever you need
+// fresh fixtures):
+//
+//	cd e2e && go run ./cmd/generate-preparams
+//
+// The output files (fixtures/node{N}_pre_params_{0,1}.json) are checked into
+// the repo so that E2E tests can seed them into each node's BadgerDB, avoiding
+// the expensive safe-prime generation at node startup.
+//
+// Each node gets its own unique pre-parameters — sharing pre-parameters across
+// nodes causes tss-lib to reject duplicate h1j values during keygen round 2.
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/bnb-chain/tss-lib/v2/ecdsa/keygen"
+)
+
+const numNodes = 3
+
+func main() {
+	outDir := "fixtures"
+	if err := os.MkdirAll(outDir, 0755); err != nil {
+		fmt.Fprintf(os.Stderr, "MkdirAll: %v\n", err)
+		os.Exit(1)
+	}
+
+	for node := 0; node < numNodes; node++ {
+		for i := 0; i < 2; i++ {
+			name := fmt.Sprintf("node%d_pre_params_%d", node, i)
+			fmt.Printf("Generating %s (this may take a minute)...\n", name)
+			start := time.Now()
+			params, err := keygen.GeneratePreParams(5 * time.Minute)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "GeneratePreParams failed: %v\n", err)
+				os.Exit(1)
+			}
+			fmt.Printf("  done in %s\n", time.Since(start).Round(time.Millisecond))
+
+			data, err := json.Marshal(params)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "json.Marshal failed: %v\n", err)
+				os.Exit(1)
+			}
+
+			out := filepath.Join(outDir, name+".json")
+			if err := os.WriteFile(out, data, 0644); err != nil {
+				fmt.Fprintf(os.Stderr, "WriteFile: %v\n", err)
+				os.Exit(1)
+			}
+			fmt.Printf("  wrote %s (%d bytes)\n", out, len(data))
+		}
+	}
+
+	fmt.Println("Done – fixture files are ready.")
+}

e2e/fixtures/node0_pre_params_0.json

@@ -0,0 +1 @@
+{"PaillierSK":{"N":25109905458989267426538098396709244292268478052491606142401927631059547309589820096686160752850851453993269954862157722785497832749590747763498048657298326998614246663577418165062348870047752715587946828268697538707240362115556588690566734463468367738035966139894847035322707324142267773323556186761137815245286107133671739805087255811013031586295490243847907364780089861950020661050135665652634308242677813852667544558248338857621161609240351900427448173029768222001575632458838215952176212099760499457142337257296697872989533954042593416781300987446759163283434834549758294673475634021828407166576040099658920257621,"LambdaN":12554952729494633713269049198354622146134239026245803071200963815529773654794910048343080376425425726996634977431078861392748916374795373881749024328649163499307123331788709082531174435023876357793973414134348769353620181057778294345283367231734183869017983069947423517661353662071133886661778093380568907622483815287472573313869981996945141830785904539940091245166628881243458083037969804010799433231686554261946474351880994949134821871878056856034307960797744282723131340242767426972453275180274077767550552046682961110257121859870941821935679930042899285272996264650765403193220755608371726020019808479577817437766,"PhiN":25109905458989267426538098396709244292268478052491606142401927631059547309589820096686160752850851453993269954862157722785497832749590747763498048657298326998614246663577418165062348870047752715587946828268697538707240362115556588690566734463468367738035966139894847035322707324142267773323556186761137815244967630574945146627739963993890283661571809079880182490333257762486916166075939608021598866463373108523892948703761989898269643743756113712068615921595488565446262680485534853944906550360548155535101104093365922220514243719741883643871359860085798570545992529301530806386441511216743452040039616959155634875532,"P":143524458835047553101016095085255056949895826978649948466577105797616555332180533290925013574387105402281835929733333752421784359939061083571826716455433064229785558292135761469424911609761812416729691598125551827903486840433842452153116452249046851945807909151606623518500660047285227205641580479626952197443,"Q":174952099891545624246275722037492867773785336989074925980254993665487939642015524340110428204917599926492759924753015206929733505545177104787005534978846592325527393681167600537844750129450531505311541565805223824571803393866867320756824675111913740791634396096620864768533462757799727920894842660876333184647},"NTildei":22412835943963862078111908299493686283867322523796478934529170772784964952620945056860020944970432480272095849840432608286715915989040425857202288548977357847057957630988277296130925536518971623587045592210563753985123309662058917060931532173474878070342140630559306731435997656015123532996152143184363151702681426378061527968292390180225169165368787046527767573423215678002965495535369233031611110323072620936359085981819260004298912908582290246574453550724871005084991242188489983582274144201962182265786537868024453172662036363090410693285522499245136535531739261546592298324862130567715636389710771626263941709361,"H1i":4143027230159654260062558934020609421301931174674214924914072966186410493683183047415402217780278454223394982469790805225827794201253462931213504320319949784920933773844274254521019516783231966765104055452373267796306254472622122142413515296451823459442932029496131131847703168532111686717499422455140204466169732750257684318205966413660522683227886097348329395828804830058812726869219939800483987248508854164574030699471887744089198617492568902067236230599525143399437581378425537979329824526536209661047260806499704315343784520331823919390221016950728942168607059464397960826695211107587894675832237441754895140412,"H2i":16301142751627308840085377519543687062320738303322826601668002311767143877911933249144980730948686884409329382404042750916395485901520373278136442005194324843665860819686312108893891323765024710641351760968345808225161879004883474594418460647046182758585529722196283979490026175670386961909484214778413756369680731687128497413812365770507199005426963438584392743642238518315927302844837391312092519867354403333147895168967865999702883424029367641553247519336028961124436086797048778971587656272370960479659069887623087017781378885595334146563079016281969975925156203511268977560676669933680316411531923970756351511879,"Alpha":22292524340989371173869503741332607843011717104933858203042975804190325529167914473354644806098099095313747334964740144791161888111609003604754957992405339765324697376340412414769197356632510965834982303250804275441273908322250510295889168036963828004885243440712595185084508748684699823468951031368898357500155410066416244885782640589393669137055690558995377042413798552759149084114391529775583138807794919164679864483620945554986129898661647360956400895142346512084369986202757677989739391001130152774667434085908354749083101880575477005942875780798079889389844548188780363000595774870843799684383929093092425285069,"Beta":2770962216928678851929416217924560903451762450669438117092852647475182234816423219528767462313090626433952649932915187495988023765357016381902332118693922257242987602625765961736196862283641832849922928381840165378181015055942791637165771035565319178674537524185703345040635205876849528771799665457939033737453245413470607269320878946798975502421336924879535055917285720594848534776087137076144631601448665169053349336761807747131181647286906858539701756378122082092360362375651103648932205969693086630010482259074924000814037290155331227185850739206768454271846978666603564404550486117607554241474094047364427706549,"P":79610937014992579855497200628829420895821793269518938776503350205195796128636858520124936967444328298168672557012256738738125828648795940111067877187220284483328903879197087929088562532305484474709157192045940218446288892140314972170500767162250704965071700709716327824459954457489740939949912652968037725779,"Q":70382402168382363526217323116248764414318909597205231445175392613372657468207798992617936029879270382235441038639607575775691905199522935660272746735426686127090004308204553576636013931192792453766899328260521207105453935195731809208020962086224952281323159476201841506569911958348685440400781893034442485939}