feat: ability to add custom tiled logs via config

Author: d-Rickyy-b

internal/certificatetransparency/ct-watcher.go

@@ -13,10 +13,11 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/google/trillian/client/backoff"
+
 	"github.com/d-Rickyy-b/certstream-server-go/internal/config"
 	"github.com/d-Rickyy-b/certstream-server-go/internal/models"
 	"github.com/d-Rickyy-b/certstream-server-go/internal/web"
-	"github.com/google/trillian/client/backoff"
 
 	ct "github.com/google/certificate-transparency-go"
 	"github.com/google/certificate-transparency-go/client"
@@ -569,7 +570,7 @@ func (w *worker) foundPrecertCallback(rawEntry *ct.RawLogEntry) {
 // certHandler takes the entries out of the entryChan channel and broadcasts them to all clients.
 // Only a single instance of the certHandler runs per certstream server.
 func certHandler(entryChan chan models.Entry) {
-	var processed int64
+	var processed uint64
 
 	for {
 		entry := <-entryChan
@@ -634,10 +635,11 @@ func getAllLogs() (loglist3.LogList, error) {
 	}
 
 	// Add manually added logs from config to the allLogs list
-	if config.AppConfig.General.AdditionalLogs == nil {
-		return allLogs, nil
-	}
+	// if config.AppConfig.General.AdditionalLogs == nil {
+	// 	return allLogs, nil
+	// }
 
+logFound:
 	for _, additionalLog := range config.AppConfig.General.AdditionalLogs {
 		customLog := loglist3.Log{
 			URL:         additionalLog.URL,
@@ -647,10 +649,16 @@ func getAllLogs() (loglist3.LogList, error) {
 		operatorFound := false
 		for _, operator := range allLogs.Operators {
 			if operator.Name == additionalLog.Operator {
-				// TODO Check if the log is already in the list
-				operator.Logs = append(operator.Logs, &customLog)
 				operatorFound = true
 
+				for _, ctlog := range operator.Logs {
+					if ctlog.URL == additionalLog.URL {
+						// Log already exists, skip it.
+						break logFound
+					}
+				}
+				// This works, since allLogs.Operators is a slice of pointers.
+				operator.Logs = append(operator.Logs, &customLog)
 				break
 			}
 		}
@@ -664,6 +672,39 @@ func getAllLogs() (loglist3.LogList, error) {
 		}
 	}
 
+	for _, additionalLog := range config.AppConfig.General.AdditionalTiledLogs {
+		customLog := loglist3.TiledLog{
+			MonitoringURL: additionalLog.URL,
+			Description:   additionalLog.Description,
+		}
+
+		operatorFound := false
+
+	tiledLogFound:
+		for _, operator := range allLogs.Operators {
+			if operator.Name == additionalLog.Operator {
+				operatorFound = true
+				for _, tl := range operator.TiledLogs {
+					if tl.MonitoringURL == additionalLog.URL {
+						// Log already exists, skip it.
+						break tiledLogFound
+					}
+				}
+				// This works, since allLogs.Operators is a slice of pointers.
+				operator.TiledLogs = append(operator.TiledLogs, &customLog)
+				break
+			}
+		}
+
+		if !operatorFound {
+			newOperator := loglist3.Operator{
+				Name:      additionalLog.Operator,
+				TiledLogs: []*loglist3.TiledLog{&customLog},
+			}
+			allLogs.Operators = append(allLogs.Operators, &newOperator)
+		}
+	}
+
 	return allLogs, nil
 }
 

internal/config/config.go

@@ -55,10 +55,11 @@ type Config struct {
 		// DisableDefaultLogs indicates whether the default logs used in Google Chrome and provided by Google should be disabled.
 		DisableDefaultLogs bool `yaml:"disable_default_logs"`
 		// AdditionalLogs contains additional logs provided by the user that can be used in addition to the default logs.
-		AdditionalLogs []LogConfig `yaml:"additional_logs"`
-		BufferSizes    BufferSizes `yaml:"buffer_sizes"`
-		DropOldLogs    *bool       `yaml:"drop_old_logs"`
-		Recovery       struct {
+		AdditionalLogs      []LogConfig `yaml:"additional_logs"`
+		AdditionalTiledLogs []LogConfig `yaml:"additional_tiled_logs"`
+		BufferSizes         BufferSizes `yaml:"buffer_sizes"`
+		DropOldLogs         *bool       `yaml:"drop_old_logs"`
+		Recovery            struct {
 			Enabled     bool   `yaml:"enabled"`
 			CTIndexFile string `yaml:"ct_index_file"`
 		} `yaml:"recovery"`
@@ -211,7 +212,7 @@ func validateConfig(config *Config) bool {
 		}
 	}
 
-	var validLogs []LogConfig
+	var validLogs, validTiledLogs []LogConfig
 	if len(config.General.AdditionalLogs) > 0 {
 		for _, ctLog := range config.General.AdditionalLogs {
 			if !URLRegex.MatchString(ctLog.URL) {
@@ -221,11 +222,25 @@ func validateConfig(config *Config) bool {
 
 			validLogs = append(validLogs, ctLog)
 		}
-	} else if len(config.General.AdditionalLogs) == 0 && config.General.DisableDefaultLogs {
-		log.Fatalln("Default logs are disabled, but no additional logs are configured. Please add at least one log to the config or enable default logs.")
+	}
+
+	if len(config.General.AdditionalTiledLogs) > 0 {
+		for _, ctLog := range config.General.AdditionalTiledLogs {
+			if !URLRegex.MatchString(ctLog.URL) {
+				log.Println("Ignoring invalid additional log URL: ", ctLog.URL)
+				continue
+			}
+
+			validTiledLogs = append(validTiledLogs, ctLog)
+		}
 	}
 
 	config.General.AdditionalLogs = validLogs
+	config.General.AdditionalTiledLogs = validTiledLogs
+
+	if len(config.General.AdditionalLogs) == 0 && len(config.General.AdditionalTiledLogs) == 0 && config.General.DisableDefaultLogs {
+		log.Fatalln("Default logs are disabled, but no additional logs are configured. Please add at least one log to the config or enable default logs.")
+	}
 
 	if config.General.BufferSizes.Websocket <= 0 {
 		config.General.BufferSizes.Websocket = 300