feat: add new config option to drop old CT logs

This fixes #51 but adds the ability to keep the current behavior.

Author: d-Rickyy-b

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 ### Added
+- Remove old CT logs as soon as they are removed from the Google CT Loglist (#60)
 ### Changed
 ### Fixed
 ### Docs

config.sample.yaml

@@ -28,3 +28,6 @@ general:
     - url: https://dodo.ct.comodo.com/
       operator: "Comodo"
       description: "Comodo Dodo"
+  # Google regularly updates the log list. If this option is set to true, the server will remove all old logs.
+  # This option defaults to true. See https://github.com/d-Rickyy-b/certstream-server-go/issues/51
+  drop_old_logs: true

internal/certificatetransparency/ct-watcher.go

@@ -55,7 +55,7 @@ func (w *Watcher) Start() {
 	}
 
 	// initialize the watcher with currently available logs
-	w.addNewlyAvailableLogs()
+	w.updateLogs()
 
 	log.Println("Started CT watcher")
 	go certHandler(w.certChan)
@@ -69,33 +69,39 @@ func (w *Watcher) Start() {
 // This method is blocking. It can be stopped by cancelling the context.
 func (w *Watcher) watchNewLogs() {
 	// Add all available logs to the watcher
-	w.addNewlyAvailableLogs()
+	w.updateLogs()
 
 	// Check for new logs once every hour
 	ticker := time.NewTicker(1 * time.Hour)
 	for {
 		select {
 		case <-ticker.C:
-			w.addNewlyAvailableLogs()
+			w.updateLogs()
 		case <-w.context.Done():
 			ticker.Stop()
 			return
 		}
 	}
 }
 
-// The transparency log list is constantly updated with new Log servers.
-// This function checks for new ct logs and adds them to the watcher.
-func (w *Watcher) addNewlyAvailableLogs() {
-	log.Println("Checking for new ct logs...")
-
+func (w *Watcher) updateLogs() {
 	// Get a list of urls of all CT logs
 	logList, err := getAllLogs()
 	if err != nil {
 		log.Println(err)
 		return
 	}
 
+	w.addNewlyAvailableLogs(logList)
+	if *config.AppConfig.General.DropOldLogs {
+		w.dropRemovedLogs(logList)
+	}
+}
+
+// addNewlyAvailableLogs checks the transparency log list for new Log servers and adds workers for those to the watcher.
+func (w *Watcher) addNewlyAvailableLogs(logList loglist3.LogList) {
+	log.Println("Checking for new ct logs...")
+
 	newCTs := 0
 
 	// Check the ct log list for new, unwatched logs
@@ -139,6 +145,12 @@ func (w *Watcher) addNewlyAvailableLogs() {
 
 	log.Printf("New ct logs found: %d\n", newCTs)
 	log.Printf("Currently monitored ct logs: %d\n", len(w.workers))
+}
+
+// dropRemovedLogs checks if any of the currently monitored logs are no longer in the log list.
+// If they are not, the CT Logs are probably no longer relevant and the corresponding workers will be stopped.
+func (w *Watcher) dropRemovedLogs(logList loglist3.LogList) {
+	removedCTs := 0
 
 	// Iterate over all workers and check if they are still in the logList
 	// If they are not, the CT Logs are probably no longer relevant.
@@ -163,9 +175,13 @@ func (w *Watcher) addNewlyAvailableLogs() {
 		// If the log is not in the loglist, stop the worker
 		if !onLogList {
 			log.Printf("Stopping worker. CT URL not found in LogList: '%s'\n", ctWorker.ctURL)
+			removedCTs++
 			ctWorker.stop()
 		}
 	}
+
+	log.Printf("Removed ct logs: %d\n", removedCTs)
+	log.Printf("Currently monitored ct logs: %d\n", len(w.workers))
 }
 
 // Stop stops the watcher.

internal/config/config.go

@@ -47,6 +47,7 @@ type Config struct {
 	}
 	General struct {
 		AdditionalLogs []LogConfig `yaml:"additional_logs"`
+		DropOldLogs    *bool       `yaml:"drop_old_logs"`
 	}
 }
 
@@ -211,5 +212,12 @@ func validateConfig(config *Config) bool {
 
 	config.General.AdditionalLogs = validLogs
 
+	// If the cleanup flag is not set, default to true
+	if config.General.DropOldLogs == nil {
+		log.Println("drop_old_logs is not set, defaulting to true")
+		defaultCleanup := true
+		config.General.DropOldLogs = &defaultCleanup
+	}
+
 	return true
 }