Skip to content

Commit 66197f5

Browse files
cmd/utils/app: implement seg index --rebuild (#21919)
`seg index --rebuild` previously `panic("not implemented")`. Now it deletes all accessor/index files and lets the existing `BuildMissed*` passes regenerate them from the data files. Removed (then rebuilt) per directory: | Ext | Dir | Data file | Rebuilt by | |-----|-----|-----------|-----------| | `.idx` | `snapshots/` | `.seg` (blocks + caplin beacon) | `br.BuildMissedIndicesIfNeed` / `caplinSnaps.BuildMissingIndices` | | `.idx` | `snapshots/caplin/` | `.seg` (caplin state) | `caplinStateSnaps.BuildMissingIndices` | | `.vi` | `snapshots/accessor/` | `.v` | `temporalDb.BuildMissedAccessors` | | `.efi` | `snapshots/accessor/` | `.ef` | `temporalDb.BuildMissedAccessors` | | `.kvi` `.bt` `.kvei` | `snapshots/domain/` | `.kv` | `temporalDb.BuildMissedAccessors` | The removed set is exactly the set the build path regenerates — every accessor type is covered (caplin state included), and only accessors are touched; data files are left intact. `.torrent` sidecars of removed accessors are deleted too. Adds `Test_removeAccessorsForRebuild` covering removal of all six accessor types + preservation of data files.
1 parent b521c4e commit 66197f5

2 files changed

Lines changed: 103 additions & 1 deletion

File tree

cmd/utils/app/snapshots_cmd.go

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2885,6 +2885,55 @@ func doDecompressSpeed(cliCtx *cli.Context) error {
28852885
return nil
28862886
}
28872887

2888+
// removeAccessorsForRebuild deletes all accessor/index files so the subsequent
2889+
// BuildMissed* passes regenerate them from the data files (.seg/.kv/.v/.ef).
2890+
func removeAccessorsForRebuild(dirs datadir.Dirs, logger log.Logger) error {
2891+
targets := []struct {
2892+
dir string
2893+
exts []string
2894+
}{
2895+
{dirs.Snap, []string{".idx"}},
2896+
{dirs.SnapCaplin, []string{".idx"}},
2897+
{dirs.SnapAccessors, []string{".vi", ".efi"}},
2898+
{dirs.SnapDomain, []string{".kvi", ".bt", ".kvei"}},
2899+
}
2900+
remove := func(fPath string) error {
2901+
if err := dir2.RemoveFile(fPath); err != nil && !errors.Is(err, os.ErrNotExist) {
2902+
return fmt.Errorf("rebuild: remove %s: %w", fPath, err)
2903+
}
2904+
logger.Info("[rebuild] removed accessor", "file", filepath.Base(fPath))
2905+
return nil
2906+
}
2907+
for _, t := range targets {
2908+
files, err := dir2.ListFiles(t.dir, t.exts...)
2909+
if err != nil {
2910+
if errors.Is(err, os.ErrNotExist) {
2911+
continue
2912+
}
2913+
return err
2914+
}
2915+
for _, fPath := range files {
2916+
if err := remove(fPath); err != nil {
2917+
return err
2918+
}
2919+
}
2920+
torrents, err := dir2.ListFiles(t.dir, ".torrent")
2921+
if err != nil {
2922+
return err
2923+
}
2924+
for _, fPath := range torrents {
2925+
base := strings.TrimSuffix(fPath, ".torrent")
2926+
if !slices.ContainsFunc(t.exts, func(ext string) bool { return strings.HasSuffix(base, ext) }) {
2927+
continue
2928+
}
2929+
if err := remove(fPath); err != nil {
2930+
return err
2931+
}
2932+
}
2933+
}
2934+
return nil
2935+
}
2936+
28882937
func doIndicesCommand(cliCtx *cli.Context, dirs datadir.Dirs) error {
28892938
logger := log.Root()
28902939
defer logger.Info("Done")
@@ -2895,7 +2944,9 @@ func doIndicesCommand(cliCtx *cli.Context, dirs datadir.Dirs) error {
28952944
defer chainDB.Close()
28962945

28972946
if rebuild {
2898-
panic("not implemented")
2947+
if err := removeAccessorsForRebuild(dirs, logger); err != nil {
2948+
return err
2949+
}
28992950
}
29002951

29012952
if err := freezeblocks.RemoveIncompatibleIndices(dirs); err != nil {

cmd/utils/app/snapshots_cmd_test.go

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1427,3 +1427,54 @@ func Test_DeleteBlockSnaps_NoBlockFilesSweepsTmp(t *testing.T) {
14271427

14281428
confirmDoesntExist(t, tmp)
14291429
}
1430+
1431+
func Test_removeAccessorsForRebuild(t *testing.T) {
1432+
dirs := datadir.New(t.TempDir())
1433+
1434+
touch := func(path string) {
1435+
f, err := os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0644)
1436+
require.NoError(t, err)
1437+
require.NoError(t, f.Close())
1438+
}
1439+
1440+
accessors := []string{
1441+
filepath.Join(dirs.Snap, "v1.0-headers.0-500.idx"),
1442+
filepath.Join(dirs.SnapCaplin, "v1.0-beaconstate.0-100.idx"),
1443+
filepath.Join(dirs.SnapAccessors, "v1.0-accounts.0-64.vi"),
1444+
filepath.Join(dirs.SnapAccessors, "v1.0-accounts.0-64.efi"),
1445+
filepath.Join(dirs.SnapDomain, "v1.0-accounts.0-64.kvi"),
1446+
filepath.Join(dirs.SnapDomain, "v1.0-accounts.0-64.bt"),
1447+
filepath.Join(dirs.SnapDomain, "v1.0-accounts.0-64.kvei"),
1448+
}
1449+
dataFiles := []string{
1450+
filepath.Join(dirs.Snap, "v1.0-headers.0-500.seg"),
1451+
filepath.Join(dirs.SnapCaplin, "v1.0-beaconstate.0-100.seg"),
1452+
filepath.Join(dirs.SnapIdx, "v1.0-accounts.0-64.ef"),
1453+
filepath.Join(dirs.SnapHistory, "v1.0-accounts.0-64.v"),
1454+
filepath.Join(dirs.SnapDomain, "v1.0-accounts.0-64.kv"),
1455+
}
1456+
for _, f := range accessors {
1457+
touch(f)
1458+
touch(f + ".torrent")
1459+
}
1460+
for _, f := range dataFiles {
1461+
touch(f)
1462+
}
1463+
1464+
orphanTorrent := filepath.Join(dirs.SnapDomain, "v1.0-storage.0-64.kvi.torrent")
1465+
touch(orphanTorrent)
1466+
dataTorrent := filepath.Join(dirs.Snap, "v1.0-headers.0-500.seg.torrent")
1467+
touch(dataTorrent)
1468+
1469+
require.NoError(t, removeAccessorsForRebuild(dirs, log.New()))
1470+
1471+
for _, f := range accessors {
1472+
confirmDoesntExist(t, f)
1473+
confirmDoesntExist(t, f+".torrent")
1474+
}
1475+
confirmDoesntExist(t, orphanTorrent)
1476+
for _, f := range dataFiles {
1477+
confirmExist(t, f)
1478+
}
1479+
confirmExist(t, dataTorrent)
1480+
}

0 commit comments

Comments
 (0)