Merge pull request #497 from bootjp/feat/metrics-etcd

raftengine: persist peers during migration to prevent split-brain leader election

Author: bootjp

internal/raftengine/etcd/migrate.go

@@ -96,6 +96,14 @@ func seedMigrationDir(tempDir string, peers []Peer, snapshotData []byte) error {
 	if err := closePersist(disk.Persist); err != nil {
 		return err
 	}
+	// Persist the peer list so the engine discovers all cluster members on
+	// first open even when the caller's FactoryConfig.Peers is empty (the
+	// common case during migration, where --raftBootstrapMembers is not
+	// repeated).  Without this file the engine falls back to a single-node
+	// configuration and every node elects itself leader independently.
+	if err := savePersistedPeers(tempDir, state.Snapshot.Metadata.Index, peers); err != nil {
+		return err
+	}
 	return nil
 }
 

internal/raftengine/etcd/migrate_test.go

@@ -98,3 +98,49 @@ func TestMigrateFSMStoreSeedsEtcdDataDir(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, []byte("one"), value)
 }
+
+func TestMigrateFSMStorePersistsSingleNodePeer(t *testing.T) {
+	sourcePath := filepath.Join(t.TempDir(), "fsm.db")
+	source, err := store.NewPebbleStore(sourcePath)
+	require.NoError(t, err)
+	require.NoError(t, source.Close())
+
+	destDataDir := filepath.Join(t.TempDir(), "raft")
+	peers := []Peer{{NodeID: 1, ID: "n1", Address: "127.0.0.1:7001"}}
+	_, err = MigrateFSMStore(sourcePath, destDataDir, peers)
+	require.NoError(t, err)
+
+	loaded, ok, err := LoadPersistedPeers(destDataDir)
+	require.NoError(t, err)
+	require.True(t, ok, "persisted peers file must exist after single-node migration")
+	require.Len(t, loaded, 1)
+	require.Equal(t, peers[0].NodeID, loaded[0].NodeID)
+}
+
+func TestMigrateFSMStorePersistsMultiNodePeers(t *testing.T) {
+	sourcePath := filepath.Join(t.TempDir(), "fsm.db")
+	source, err := store.NewPebbleStore(sourcePath)
+	require.NoError(t, err)
+	require.NoError(t, source.Close())
+
+	destDataDir := filepath.Join(t.TempDir(), "raft")
+	peers := []Peer{
+		{NodeID: 1, ID: "n1", Address: "127.0.0.1:7001"},
+		{NodeID: 2, ID: "n2", Address: "127.0.0.1:7002"},
+		{NodeID: 3, ID: "n3", Address: "127.0.0.1:7003"},
+	}
+	_, err = MigrateFSMStore(sourcePath, destDataDir, peers)
+	require.NoError(t, err)
+
+	// Persisted peers must exist so the engine discovers all cluster members
+	// even when FactoryConfig.Peers is empty (the common post-migration case).
+	loaded, ok, err := LoadPersistedPeers(destDataDir)
+	require.NoError(t, err)
+	require.True(t, ok, "persisted peers file must exist after migration")
+	require.Len(t, loaded, 3)
+	for i, peer := range loaded {
+		require.Equal(t, peers[i].NodeID, peer.NodeID)
+		require.Equal(t, peers[i].ID, peer.ID)
+		require.Equal(t, peers[i].Address, peer.Address)
+	}
+}

multiraft_runtime.go

@@ -140,6 +140,8 @@ func detectRaftEngineFromDataDir(dir string) (raftEngineType, bool, error) {
 		return "", false, err
 	}
 	etcdArtifacts, err := hasRaftArtifacts(dir,
+		"wal",
+		"snap",
 		filepath.Join("member", "wal"),
 		filepath.Join("member", "snap"),
 		"etcd-raft-state.bin",

multiraft_runtime_test.go

@@ -224,4 +224,31 @@ func TestEnsureRaftEngineDataDir(t *testing.T) {
 		require.True(t, ok)
 		require.Equal(t, raftEngineEtcd, engineType)
 	})
+
+	t.Run("detects bare wal dir as etcd artifact", func(t *testing.T) {
+		dir := t.TempDir()
+		require.NoError(t, os.MkdirAll(filepath.Join(dir, "wal"), 0o755))
+		engineType, ok, err := detectRaftEngineFromDataDir(dir)
+		require.NoError(t, err)
+		require.True(t, ok)
+		require.Equal(t, raftEngineEtcd, engineType)
+	})
+
+	t.Run("detects bare snap dir as etcd artifact", func(t *testing.T) {
+		dir := t.TempDir()
+		require.NoError(t, os.MkdirAll(filepath.Join(dir, "snap"), 0o755))
+		engineType, ok, err := detectRaftEngineFromDataDir(dir)
+		require.NoError(t, err)
+		require.True(t, ok)
+		require.Equal(t, raftEngineEtcd, engineType)
+	})
+
+	t.Run("detects mixed engine artifacts with bare wal", func(t *testing.T) {
+		dir := t.TempDir()
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "raft.db"), []byte("dummy"), 0o600))
+		require.NoError(t, os.MkdirAll(filepath.Join(dir, "wal"), 0o755))
+		_, _, err := detectRaftEngineFromDataDir(dir)
+		require.Error(t, err)
+		require.ErrorIs(t, err, ErrMixedRaftEngineArtifacts)
+	})
 }

scripts/engine-migrate.sh

@@ -288,7 +288,16 @@ sudo -n "$MIGRATE_BIN" \
   -peers "$PEERS"
 
 echo "  moving etcd artifacts into place"
-sudo -n mv "${migrate_dest}/data/member" "${node_dir}/member"
+for artifact in wal snap etcd-raft-peers.bin; do
+  if sudo -n test -e "${migrate_dest}/data/${artifact}"; then
+    sudo -n mv "${migrate_dest}/data/${artifact}" "${node_dir}/${artifact}"
+  fi
+done
+# Fallback: handle legacy member/ directory layout produced by older
+# migration binaries so WAL/snap data is never silently dropped.
+if sudo -n test -d "${migrate_dest}/data/member"; then
+  sudo -n mv "${migrate_dest}/data/member" "${node_dir}/member"
+fi
 sudo -n rm -rf "$migrate_dest"
 
 echo "  archiving hashicorp raft artifacts"