Skip to content

Commit c13fce5

Browse files
authored
[POSIX] Migrate allows writing checkpoint (#906)
This is a pragmatic way to support setting up mirrors (#576) from POSIX logs. Confirmed this works by cloning SumDB to a POSIX TLogTiles log: ```shell go run github.com/transparency-dev/incubator/tree/main/sumdb/cmd@main --listen=":8089" & go run ./cmd/experimental/migrate/posix \ --storage_dir ~/log-clones/sumdb \ --source_url http://localhost:8089/ \ --save_checkpoint echo sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8 > ~/.go.sum.vkey go run github.com/transparency-dev/tessera/cmd/fsck@main \ --storage_url file:///${HOME}/log-clones/sumdb/ \ --public_key ~/.go.sum.vkey \ --origin "go.sum database tree" ```
1 parent 10c5ad5 commit c13fce5

1 file changed

Lines changed: 14 additions & 4 deletions

File tree

  • cmd/experimental/migrate/posix

cmd/experimental/migrate/posix/main.go

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,21 +22,24 @@ import (
2222
"flag"
2323
"net/url"
2424
"os"
25+
"path/filepath"
2526
"strconv"
2627
"strings"
2728

2829
"log/slog"
2930

3031
"github.com/transparency-dev/tessera"
32+
"github.com/transparency-dev/tessera/api/layout"
3133
"github.com/transparency-dev/tessera/client"
3234
"github.com/transparency-dev/tessera/storage/posix"
3335
)
3436

3537
var (
36-
storageDir = flag.String("storage_dir", "", "Root directory to store log data.")
37-
sourceURL = flag.String("source_url", "", "Base URL for the source log.")
38-
numWorkers = flag.Uint("num_workers", 30, "Number of migration worker goroutines.")
39-
slogLevel = flag.Int("slog_level", 0, "The cut-off threshold for structured logging. Default is 0 (INFO). See https://pkg.go.dev/log/slog#Level for other levels.")
38+
storageDir = flag.String("storage_dir", "", "Root directory to store log data.")
39+
sourceURL = flag.String("source_url", "", "Base URL for the source log.")
40+
numWorkers = flag.Uint("num_workers", 30, "Number of migration worker goroutines.")
41+
slogLevel = flag.Int("slog_level", 0, "The cut-off threshold for structured logging. Default is 0 (INFO). See https://pkg.go.dev/log/slog#Level for other levels.")
42+
saveCheckpoint = flag.Bool("save_checkpoint", false, "Set to true to write the checkpoint used during migration. Useful for mirrors.")
4043
)
4144

4245
func main() {
@@ -87,4 +90,11 @@ func main() {
8790
slog.ErrorContext(ctx, "Migrate failed", slog.Any("error", err))
8891
os.Exit(1)
8992
}
93+
94+
if *saveCheckpoint {
95+
if err := os.WriteFile(filepath.Join(*storageDir, layout.CheckpointPath), sourceCP, 0o644); err != nil {
96+
slog.ErrorContext(ctx, "Failed to write checkpoint", slog.Any("error", err))
97+
os.Exit(1)
98+
}
99+
}
90100
}

0 commit comments

Comments
 (0)