Merge branch 'master' into remove-old-ctlogs

Author: d-Rickyy-b

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Added
 - Remove old CT logs as soon as they are removed from the Google CT Loglist (#60)
+- New configuration for buffer sizes (#58)
 ### Changed
 ### Fixed
 ### Docs
@@ -18,9 +19,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Support for non-browsers by implementing server initiated heartbeats (#39)
 - Start new ct-watchers as new ct logs become available (#42)
 - More logging to document currently watched logs (03d878e)
+ 
 ### Changed
 - Changed log output to be better grepable (5c055cc)
 - Update ct log update interval to once per hour instead of once per 6 hours as previously (9b6e77d)
+
 ### Fixed
 - Fixed a possible race condition when accessing metrics
 

config.sample.yaml

@@ -28,6 +28,15 @@ general:
     - url: https://dodo.ct.comodo.com/
       operator: "Comodo"
       description: "Comodo Dodo"
+  # To optimize the performance of the server, you can overwrite the size of different buffers
+  # For low CPU, low memory machines, you should reduce the buffer sizes to save memory in case the CPU is maxed.
+  buffer_sizes:
+    # Buffer for each websocket connection
+    websocket: 300
+    # Buffer for each CT log connection
+    ctlog: 1000
+    # Combined buffer for the broadcast manager
+    broadcastmanager: 10000
   # Google regularly updates the log list. If this option is set to true, the server will remove all old logs.
   # This option defaults to true. See https://github.com/d-Rickyy-b/certstream-server-go/issues/51
   drop_old_logs: true

internal/certificatetransparency/ct-watcher.go

@@ -292,7 +292,7 @@ func (w *worker) runWorker(ctx context.Context) error {
 		Matcher:     scanner.MatchAll{},
 		PrecertOnly: false,
 		NumWorkers:  1,
-		BufferSize:  1000,
+		BufferSize:  config.AppConfig.General.BufferSizes.CTLog,
 	})
 
 	scanErr := certScanner.Scan(ctx, w.foundCertCallback, w.foundPrecertCallback)

internal/config/config.go

@@ -31,6 +31,12 @@ type LogConfig struct {
 	Description string `yaml:"description"`
 }
 
+type BufferSizes struct {
+	Websocket        int `yaml:"websocket"`
+	CTLog            int `yaml:"ctlog"`
+	BroadcastManager int `yaml:"broadcastmanager"`
+}
+
 type Config struct {
 	Webserver struct {
 		ServerConfig       `yaml:",inline"`
@@ -47,6 +53,7 @@ type Config struct {
 	}
 	General struct {
 		AdditionalLogs []LogConfig `yaml:"additional_logs"`
+		BufferSizes    BufferSizes `yaml:"buffer_sizes"`
 		DropOldLogs    *bool       `yaml:"drop_old_logs"`
 	}
 }
@@ -212,12 +219,23 @@ func validateConfig(config *Config) bool {
 
 	config.General.AdditionalLogs = validLogs
 
-	// If the cleanup flag is not set, default to true
+	if config.General.BufferSizes.Websocket <= 0 {
+		config.General.BufferSizes.Websocket = 300
+	}
+
+	if config.General.BufferSizes.CTLog <= 0 {
+		config.General.BufferSizes.CTLog = 1000
+	}
+
+	if config.General.BufferSizes.BroadcastManager <= 0 {
+		config.General.BufferSizes.BroadcastManager = 10000
+
+  // If the cleanup flag is not set, default to true
 	if config.General.DropOldLogs == nil {
 		log.Println("drop_old_logs is not set, defaulting to true")
 		defaultCleanup := true
 		config.General.DropOldLogs = &defaultCleanup
-	}
+  }
 
 	return true
 }

internal/web/server.go

@@ -168,7 +168,7 @@ func upgradeConnection(w http.ResponseWriter, r *http.Request) (*websocket.Conn,
 
 // setupClient initializes a client struct and starts the broadcastHandler and websocket listener.
 func setupClient(connection *websocket.Conn, subscriptionType SubscriptionType, name string) {
-	c := newClient(connection, subscriptionType, name, 300)
+	c := newClient(connection, subscriptionType, name, config.AppConfig.General.BufferSizes.Websocket)
 	go c.broadcastHandler()
 	go c.listenWebsocket()
 
@@ -275,7 +275,7 @@ func NewWebsocketServer(networkIf string, port int, certPath, keyPath string) *W
 	setupWebsocketRoutes(server.routes)
 	server.initServer()
 
-	ClientHandler.Broadcast = make(chan models.Entry, 10_000)
+	ClientHandler.Broadcast = make(chan models.Entry, config.AppConfig.General.BufferSizes.BroadcastManager)
 	go ClientHandler.broadcaster()
 
 	return server