feat: add feature to create initial index file

This is useful in order to not start pulling each log after the first run at index 0 but still have newly discovered logs download from 0.

Author: d-Rickyy-b

cmd/certstream-server-go/main.go

@@ -13,6 +13,7 @@ import (
 func main() {
 	configFile := flag.String("config", "config.yml", "path to the config file")
 	versionFlag := flag.Bool("version", false, "Print the version and exit")
+	createIndexFile := flag.Bool("create-index-file", false, "Create the ct_index.json based on current STHs")
 	flag.Parse()
 
 	if *versionFlag {
@@ -21,6 +22,22 @@ func main() {
 	}
 
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
+
+	// If the user only wants to create the index file, we don't need to start the server
+	if *createIndexFile {
+		conf, readConfErr := config.ReadConfig(*configFile)
+		if readConfErr != nil {
+			log.Fatalf("Error while reading config: %v", readConfErr)
+		}
+		cs := certstream.NewRawCertstream(conf)
+
+		createErr := cs.CreateIndexFile()
+		if createErr != nil {
+			log.Fatalf("Error while creating index file: %v", createErr)
+		}
+		return
+	}
+
 	log.Printf("Starting certstream-server-go v%s\n", config.Version)
 
 	cs, err := certstream.NewCertstreamFromConfigFile(*configFile)

config.sample.yaml

@@ -47,7 +47,10 @@ general:
   # Options for resuming certificate downloads after restart
   recovery:
     # If enabled, the server will resume downloading certificates from the last processed and stored index for each log.
-    # If there is no ct_index_file or for a specific log there is no index entry, the server will start from the latest STH.
+    # If there is no ct_index_file or for a specific log there is no index entry, the server will start from index 0.
+    # Be aware that this leads to a massive number of certificates being downloaded.
+    # Depending on your server's performance and network connection, this could be up to 10.000 certificates per second.
+    # Make sure your infrastructure can handle this!
     enabled: true
     # Path to the file where indices are stored. Be aware that a temp file in the same path with the same name and ".tmp" as suffix will be created.
     # If there are no write permissions to the path, the server will not be able to store the indices.

internal/certificatetransparency/ct-watcher.go

@@ -214,6 +214,48 @@ func (w *Watcher) Stop() {
 	w.cancelFunc()
 }
 
+// CreateIndexFile creates a ct_index.json file based on the current STHs of all availble logs.
+func (w *Watcher) CreateIndexFile(filePath string) error {
+	logs, err := getAllLogs()
+	if err != nil {
+		return err
+	}
+
+	w.context, w.cancelFunc = context.WithCancel(context.Background())
+	log.Println("Fetching current STH for all logs...")
+	for _, operator := range logs.Operators {
+		// Iterate over each log of the operator
+		for _, transparencyLog := range operator.Logs {
+			// Check if the log is already being watched
+			metrics.Init(operator.Name, transparencyLog.URL)
+			log.Println("Fetching STH for", transparencyLog.URL)
+
+			hc := http.Client{Timeout: 5 * time.Second}
+			jsonClient, e := client.New(transparencyLog.URL, &hc, jsonclient.Options{UserAgent: userAgent})
+			if e != nil {
+				log.Printf("Error creating JSON client: %s\n", e)
+				continue
+			}
+
+			sth, getSTHerr := jsonClient.GetSTH(w.context)
+			if getSTHerr != nil {
+				// TODO this can happen due to a 429 error. We should retry the request
+				log.Printf("Could not get STH for '%s': %s\n", transparencyLog.URL, getSTHerr)
+				continue
+			}
+
+			metrics.index[transparencyLog.URL] = int64(sth.TreeSize)
+		}
+	}
+	w.cancelFunc()
+
+	tempFilePath := fmt.Sprintf("%s.tmp", filePath)
+	metrics.SaveCertIndexes(tempFilePath, filePath)
+	log.Println("Index file saved to", filePath)
+
+	return nil
+}
+
 // A worker processes a single CT log.
 type worker struct {
 	name         string

internal/certstream/certstream.go

@@ -23,6 +23,13 @@ type Certstream struct {
 	config        config.Config
 }
 
+func NewRawCertstream(config config.Config) *Certstream {
+	cs := Certstream{}
+	cs.config = config
+
+	return &cs
+}
+
 // NewCertstreamServer creates a new Certstream server from a config struct.
 func NewCertstreamServer(config config.Config) (*Certstream, error) {
 	cs := Certstream{}
@@ -108,6 +115,16 @@ func (cs *Certstream) Stop() {
 	}
 }
 
+// CreateIndexFile creates the index file for the certificate transparency logs.
+// It gets only called when the CLI flag --create-index-file is set.
+func (cs *Certstream) CreateIndexFile() error {
+	// If there is no watcher initialized, create a new one
+	if cs.watcher == nil {
+		cs.watcher = &certificatetransparency.Watcher{}
+	}
+	return cs.watcher.CreateIndexFile(cs.config.General.Recovery.CTIndexFile)
+}
+
 // signalHandler listens for signals in order to gracefully shut down the server.
 // Executes the callback function when a signal is received.
 func signalHandler(signals chan os.Signal, callback func()) {