diff --git a/CHANGELOG.md b/CHANGELOG.md index ec6412b..c7bafab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- New configuration for buffer sizes (#58) ### Changed ### Fixed ### Docs @@ -17,9 +18,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support for non-browsers by implementing server initiated heartbeats (#39) - Start new ct-watchers as new ct logs become available (#42) - More logging to document currently watched logs (03d878e) + ### Changed - Changed log output to be better grepable (5c055cc) - Update ct log update interval to once per hour instead of once per 6 hours as previously (9b6e77d) + ### Fixed - Fixed a possible race condition when accessing metrics diff --git a/config.sample.yaml b/config.sample.yaml index 0f80c9f..30744a7 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -28,3 +28,12 @@ general: - url: https://dodo.ct.comodo.com/ operator: "Comodo" description: "Comodo Dodo" + # To optimize the performance of the server, you can overwrite the size of different buffers + # For low CPU, low memory machines, you should reduce the buffer sizes to save memory in case the CPU is maxed. + buffer_sizes: + # Buffer for each websocket connection + websocket: 300 + # Buffer for each CT log connection + ctlog: 1000 + # Combined buffer for the broadcast manager + broadcastmanager: 10000 diff --git a/internal/certificatetransparency/ct-watcher.go b/internal/certificatetransparency/ct-watcher.go index ab6a0df..9218465 100644 --- a/internal/certificatetransparency/ct-watcher.go +++ b/internal/certificatetransparency/ct-watcher.go @@ -238,7 +238,7 @@ func (w *worker) runWorker(ctx context.Context) error { Matcher: scanner.MatchAll{}, PrecertOnly: false, NumWorkers: 1, - BufferSize: 1000, + BufferSize: config.AppConfig.General.BufferSizes.CTLog, }) scanErr := certScanner.Scan(ctx, w.foundCertCallback, w.foundPrecertCallback) diff --git a/internal/config/config.go b/internal/config/config.go index 1993983..0010acc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -31,6 +31,12 @@ type LogConfig struct { Description string `yaml:"description"` } +type BufferSizes struct { + Websocket int `yaml:"websocket"` + CTLog int `yaml:"ctlog"` + BroadcastManager int `yaml:"broadcastmanager"` +} + type Config struct { Webserver struct { ServerConfig `yaml:",inline"` @@ -47,6 +53,7 @@ type Config struct { } General struct { AdditionalLogs []LogConfig `yaml:"additional_logs"` + BufferSizes BufferSizes `yaml:"buffer_sizes"` } } @@ -211,5 +218,17 @@ func validateConfig(config *Config) bool { config.General.AdditionalLogs = validLogs + if config.General.BufferSizes.Websocket <= 0 { + config.General.BufferSizes.Websocket = 300 + } + + if config.General.BufferSizes.CTLog <= 0 { + config.General.BufferSizes.CTLog = 1000 + } + + if config.General.BufferSizes.BroadcastManager <= 0 { + config.General.BufferSizes.BroadcastManager = 10000 + } + return true } diff --git a/internal/web/server.go b/internal/web/server.go index 8ffb62a..b2e42aa 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -168,7 +168,7 @@ func upgradeConnection(w http.ResponseWriter, r *http.Request) (*websocket.Conn, // setupClient initializes a client struct and starts the broadcastHandler and websocket listener. func setupClient(connection *websocket.Conn, subscriptionType SubscriptionType, name string) { - c := newClient(connection, subscriptionType, name, 300) + c := newClient(connection, subscriptionType, name, config.AppConfig.General.BufferSizes.Websocket) go c.broadcastHandler() go c.listenWebsocket() @@ -275,7 +275,7 @@ func NewWebsocketServer(networkIf string, port int, certPath, keyPath string) *W setupWebsocketRoutes(server.routes) server.initServer() - ClientHandler.Broadcast = make(chan models.Entry, 10_000) + ClientHandler.Broadcast = make(chan models.Entry, config.AppConfig.General.BufferSizes.BroadcastManager) go ClientHandler.broadcaster() return server