perf(sandbox): reduce memory dirtying from journald and envd logging

packages/envd/internal/host/mmds.go

@@ -135,32 +135,61 @@
 	return opts.AccessTokenHash, nil
 }
 
+// MMDS poll backoff: start fast for the common happy path (MMDS up at boot),
+// then double on each failure up to mmdsMaxBackoff so a persistently broken
+// endpoint doesn't keep firing HTTP requests every 50ms.
+const (
+	mmdsInitialBackoff = 50 * time.Millisecond
+	mmdsMaxBackoff     = 1 * time.Second
+)
+
 func PollForMMDSOpts(ctx context.Context, mmdsChan chan<- *MMDSOpts, envVars *utils.Map[string, string]) {
 	httpClient := &http.Client{}
 	defer httpClient.CloseIdleConnections()
 
-	ticker := time.NewTicker(50 * time.Millisecond)
-	defer ticker.Stop()
+	// Log only the first failure of each kind. Tagged as syslog WARNING (<4>)
+	// since MMDS hiccups are expected during early boot and recoverable.
+	var loggedTokenErr, loggedOptsErr bool
+
+	backoff := mmdsInitialBackoff
+	timer := time.NewTimer(backoff)
+	defer timer.Stop()
+
+	advanceBackoff := func() {
+		backoff *= 2
+		if backoff > mmdsMaxBackoff {
+			backoff = mmdsMaxBackoff
+		}
+		timer.Reset(backoff)
+	}
 
 	for {
 		select {
 		case <-ctx.Done():
-			fmt.Fprintf(os.Stderr, "context cancelled while waiting for mmds opts")
+			fmt.Fprintf(os.Stderr, "<4>context cancelled while waiting for mmds opts\n")
 
 			return
-		case <-ticker.C:
+		case <-timer.C:
 			token, err := getMMDSToken(ctx, httpClient)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "error getting mmds token: %v\n", err)
+				if !loggedTokenErr {
+					fmt.Fprintf(os.Stderr, "<4>error getting mmds token (suppressing further failures): %v\n", err)
+					loggedTokenErr = true
+				}
 
+				advanceBackoff()
 				continue
 			}
 
 			mmdsOpts, err := getMMDSOpts(ctx, httpClient, token)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "error getting mmds opts: %v\n", err)
+				if !loggedOptsErr {
+					fmt.Fprintf(os.Stderr, "<4>error getting mmds opts (suppressing further failures): %v\n", err)
+					loggedOptsErr = true
+				}
 
+				advanceBackoff()
 				continue
 			}
 
 			envVars.Store("E2B_SANDBOX_ID", mmdsOpts.SandboxID)

packages/envd/internal/logs/exporter/exporter.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"log"
 	"net/http"
 	"os"
 	"sync"
@@ -13,13 +12,32 @@ import (
 	"github.com/e2b-dev/infra/packages/envd/internal/host"
 )
 
-const ExporterTimeout = 10 * time.Second
+const (
+	ExporterTimeout = 10 * time.Second
+
+	// maxBufferedBytes caps the total in-memory size of the log queue, so the
+	// queue stays bounded even when the collector is unreachable for a long
+	// time (e.g. before MMDS opts arrive on boot). Oldest logs are dropped on
+	// over-cap so the most recent — and most useful for whatever just
+	// happened — are the ones the orchestrator gets once it can be reached.
+	maxBufferedBytes = 4 << 20 // 4 MiB
+
+	// Exporter send-failure backoff: each failure waits sendInitialBackoff,
+	// doubling up to sendMaxBackoff, before any further send is attempted.
+	// During the cooldown window logs are dropped (we already report the
+	// outage to journald and the orchestrator was supposed to receive them
+	// anyway). Resets to the initial value on the next successful send.
+	sendInitialBackoff = 1 * time.Second
+	sendMaxBackoff     = 5 * time.Minute
+)
 
 type HTTPExporter struct {
-	client   http.Client
-	logs     [][]byte
-	isNotFC  bool
-	mmdsOpts *host.MMDSOpts
+	client        http.Client
+	logs          [][]byte
+	bufferedBytes int
+	isNotFC       bool
+	verbose       bool
+	mmdsOpts      *host.MMDSOpts
 
 	// Concurrency coordination
 	triggers  chan struct{}
@@ -28,13 +46,14 @@ type HTTPExporter struct {
 	startOnce sync.Once
 }
 
-func NewHTTPLogsExporter(ctx context.Context, isNotFC bool, mmdsChan <-chan *host.MMDSOpts) *HTTPExporter {
+func NewHTTPLogsExporter(ctx context.Context, isNotFC bool, verbose bool, mmdsChan <-chan *host.MMDSOpts) *HTTPExporter {
 	exporter := &HTTPExporter{
 		client: http.Client{
 			Timeout: ExporterTimeout,
 		},
 		triggers:  make(chan struct{}, 1),
 		isNotFC:   isNotFC,
+		verbose:   verbose,
 		startOnce: sync.Once{},
 		mmdsOpts: &host.MMDSOpts{
 			SandboxID:            "unknown",
@@ -69,7 +88,12 @@ func (w *HTTPExporter) sendInstanceLogs(ctx context.Context, logs []byte, addres
 	return nil
 }
 
-func printLog(logs []byte) {
+func (w *HTTPExporter) printLog(logs []byte) {
+	// Only emit on -verbose. Inside FC stdout flows into journald, which is
+	// what this PR is trying to keep clean during send outages.
+	if !w.verbose {
+		return
+	}
 	fmt.Fprintf(os.Stdout, "%v", string(logs))
 }
 
@@ -95,6 +119,16 @@ func (w *HTTPExporter) listenForMMDSOptsAndStart(ctx context.Context, mmdsChan <
 }
 
 func (w *HTTPExporter) start(ctx context.Context) {
+	// Suppress repeated failures so a wedged collector doesn't 1:1-amplify
+	// envd logs into journald.
+	var loggedJSONErr, loggedSendErr bool
+
+	// Exponential backoff between send attempts after a failure, so a
+	// persistently broken collector doesn't keep wasting ExporterTimeout
+	// (10s) per log line.
+	var cooldownUntil time.Time
+	cooldown := sendInitialBackoff
+
 	for range w.triggers {
 		logs := w.getAllLogs()
 
@@ -103,8 +137,10 @@ func (w *HTTPExporter) start(ctx context.Context) {
 		}
 
 		if w.isNotFC {
-			for _, log := range logs {
-				fmt.Fprintf(os.Stdout, "%v", string(log))
+			if w.verbose {
+				for _, log := range logs {
+					fmt.Fprintf(os.Stdout, "%v", string(log))
+				}
 			}
 
 			continue
@@ -115,21 +151,42 @@ func (w *HTTPExporter) start(ctx context.Context) {
 			logLineWithOpts, err := w.mmdsOpts.AddOptsToJSON(logLine)
 			w.mmdsLock.RUnlock()
 			if err != nil {
-				log.Printf("error adding instance logging options (%+v) to JSON (%+v) with logs : %v\n", w.mmdsOpts, logLine, err)
+				if !loggedJSONErr {
+					fmt.Fprintf(os.Stderr, "<4>error adding instance logging options (%+v) to JSON (%+v) with logs (suppressing further failures): %v\n", w.mmdsOpts, logLine, err)
+					loggedJSONErr = true
+				}
 
-				printLog(logLine)
+				w.printLog(logLine)
 
 				continue
 			}
 
+			if time.Now().Before(cooldownUntil) {
+				// Skip send during cooldown; the log is dropped.
+				continue
+			}
+
 			err = w.sendInstanceLogs(ctx, logLineWithOpts, w.mmdsOpts.LogsCollectorAddress)
 			if err != nil {
-				log.Printf("error sending instance logs: %+v", err)
+				if !loggedSendErr {
+					fmt.Fprintf(os.Stderr, "<4>error sending instance logs (suppressing further failures): %v\n", err)
+					loggedSendErr = true
+				}
+
+				cooldownUntil = time.Now().Add(cooldown)
+				cooldown *= 2
+				if cooldown > sendMaxBackoff {
+					cooldown = sendMaxBackoff
+				}
 
-				printLog(logLine)
+				w.printLog(logLine)
 
 				continue
 			}
+
+			cooldown = sendInitialBackoff
+			cooldownUntil = time.Time{}
+			loggedSendErr = false
 		}
 	}
 }
@@ -158,6 +215,7 @@ func (w *HTTPExporter) getAllLogs() [][]byte {
 
 	logs := w.logs
 	w.logs = nil
+	w.bufferedBytes = 0
 
 	return logs
 }
@@ -166,6 +224,16 @@ func (w *HTTPExporter) addLogs(logs []byte) {
 	w.logLock.Lock()
 	defer w.logLock.Unlock()
 
+	// Drop oldest so total buffered size stays at or below maxBufferedBytes.
+	// Most recent logs are kept (those are usually the ones we want to see
+	// after a long buffering window, e.g. early boot before MMDS is up).
+	for w.bufferedBytes+len(logs) > maxBufferedBytes && len(w.logs) > 0 {
+		w.bufferedBytes -= len(w.logs[0])
+		w.logs[0] = nil // let GC reclaim the dropped log content
+		w.logs = w.logs[1:]
+	}
+
+	w.bufferedBytes += len(logs)
 	w.logs = append(w.logs, logs)
 
 	w.resumeProcessing()

packages/envd/internal/logs/logger.go

@@ -12,16 +12,20 @@ import (
 	"github.com/e2b-dev/infra/packages/envd/internal/logs/exporter"
 )
 
-func NewLogger(ctx context.Context, isNotFC bool, mmdsChan <-chan *host.MMDSOpts) *zerolog.Logger {
+func NewLogger(ctx context.Context, isNotFC bool, verbose bool, mmdsChan <-chan *host.MMDSOpts) *zerolog.Logger {
 	zerolog.TimestampFieldName = "timestamp"
 	zerolog.TimeFieldFormat = time.RFC3339Nano
 
 	exporters := []io.Writer{}
 
-	if isNotFC {
+	if !isNotFC {
+		exporters = append(exporters, exporter.NewHTTPLogsExporter(ctx, isNotFC, verbose, mmdsChan))
+	}
+	// Stdout is opt-in via -verbose. Inside FC stdout flows into journald and
+	// dirties guest pages on every snapshot, so we keep it off by default and
+	// rely on the HTTP exporter to ship debug logs to the orchestrator.
+	if verbose {
 		exporters = append(exporters, os.Stdout)
-	} else {
-		exporters = append(exporters, exporter.NewHTTPLogsExporter(ctx, isNotFC, mmdsChan), os.Stdout)
 	}
 
 	l := zerolog.

packages/envd/internal/services/process/handler/handler.go

@@ -221,7 +221,7 @@ func New(
 				}
 
 				if readErr != nil {
-					fmt.Fprintf(os.Stderr, "error reading from pty: %s\n", readErr)
+					logger.Error().Err(readErr).Msg("error reading from pty")
 
 					break
 				}
@@ -269,7 +269,7 @@ func New(
 				}
 
 				if readErr != nil {
-					fmt.Fprintf(os.Stderr, "error reading from stdout: %s\n", readErr)
+					logger.Error().Err(readErr).Msg("error reading from stdout")
 
 					break
 				}
@@ -313,7 +313,7 @@ func New(
 				}
 
 				if readErr != nil {
-					fmt.Fprintf(os.Stderr, "error reading from stderr: %s\n", readErr)
+					logger.Error().Err(readErr).Msg("error reading from stderr")
 
 					break
 				}

packages/envd/main.go

@@ -56,6 +56,7 @@ var (
 	versionFlag bool
 	commitFlag  bool
 	cgroupRoot  string
+	verbose     bool
 )
 
 func parseFlags() {
@@ -94,6 +95,13 @@ func parseFlags() {
 		"cgroup root directory",
 	)
 
+	flag.BoolVar(
+		&verbose,
+		"verbose",
+		false,
+		"write envd logs to stdout (inside FC this would dirty journald pages; HTTP exporter still ships full debug regardless)",
+	)
+
 	flag.Parse()
 }
 
@@ -159,7 +167,7 @@ func main() {
 		go host.PollForMMDSOpts(ctx, mmdsChan, defaults.EnvVars)
 	}
 
-	l := logs.NewLogger(ctx, isNotFC, mmdsChan)
+	l := logs.NewLogger(ctx, isNotFC, verbose, mmdsChan)
 
 	m := chi.NewRouter()
 

packages/envd/pkg/version.go

@@ -1,3 +1,3 @@
 package pkg
 
-const Version = "0.5.20"
+const Version = "0.5.24"

packages/orchestrator/pkg/template/build/core/rootfs/files/journald.conf.tpl

@@ -0,0 +1,12 @@
+{{- /*gotype:github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/rootfs.templateModel*/ -}}
+{{ .WriteFile "etc/systemd/journald.conf.d/e2b.conf" 0o644 }}
+
+[Journal]
+Storage=persistent
+SystemMaxUse=8M
+SystemMaxFileSize=2M
+MaxLevelStore=warning
+MaxLevelConsole=warning
+MaxLevelKMsg=warning
+MaxLevelWall=emerg
+ForwardToSyslog=no

packages/orchestrator/pkg/template/build/core/rootfs/rootfs_test.go

@@ -90,7 +90,7 @@ func TestAdditionalOCILayers(t *testing.T) {
 
 		keysIter := maps.Keys(actualFiles)
 		keys := slices.Collect(keysIter)
-		assert.Len(t, keys, 13)
+		assert.Len(t, keys, 14)
 		assert.Equal(t, "e2b.local", actualFiles["etc/hostname"])
 		assert.Equal(t, "nameserver 8.8.8.8", actualFiles["etc/resolv.conf"])