fix(cloud): make push payload limit configurable

Alan-TheGentleman · web-flow · commit ad737f4c04f3 · 2026-05-20T10:17:03.000+02:00
Closes #316
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -21,6 +21,10 @@ Breaking changes are always marked with a `type:breaking-change` label and docum
 
 <!-- Changes that are merged but not yet released are tracked here until the next tag. -->
 
+### Cloud sync
+
+- **fix(cloud):** make chunk and mutation push payload limits configurable with `ENGRAM_CLOUD_MAX_PUSH_BYTES` while preserving the 8 MiB default.
+
 ### Pi package (`pi-engram`)
 
 - **fix(plugin):** allow `mem_session_summary` to accept an explicit `project` fallback when automatic project detection is unavailable.
@@ -49,7 +53,7 @@ New and updated routes registered in `internal/cloud/dashboard/dashboard.go`:
 Background mutation-based replication for `engram serve` and `engram mcp`:
 
 - **feat(autosync):** `internal/cloud/autosync.Manager` — lease-guarded background push/pull goroutine enabled by `ENGRAM_CLOUD_AUTOSYNC=1` + `ENGRAM_CLOUD_TOKEN` + `ENGRAM_CLOUD_SERVER`
-- **feat(cloudserver):** add `POST /sync/mutations/push` (batch up to 100 mutations, 8 MiB body cap, per-project auth + pause gate returning HTTP 409 `sync-paused`)
+- **feat(cloudserver):** add `POST /sync/mutations/push` (batch up to 100 mutations, configurable body cap defaulting to 8 MiB, per-project auth + pause gate returning HTTP 409 `sync-paused`)
 - **feat(cloudserver):** add `GET /sync/mutations/pull?since_seq=N&limit=M` (server-side filtered by enrolled projects; fail-closed when `EnrolledProjectsProvider` not implemented)
 - **feat(autosync):** phases: `idle`, `pushing`, `pulling`, `healthy`, `push_failed`, `pull_failed`, `backoff`, `disabled`
 - **feat(autosync):** reason codes: `transport_failed`, `auth_required`, `policy_forbidden`, `server_unsupported`, `internal_error`, `sync-paused`
diff --git a/DOCS.md b/DOCS.md
@@ -548,6 +548,7 @@ Cloud runtime envs for `engram cloud serve`:
 | `ENGRAM_DATABASE_URL`           | yes                      | Postgres DSN for cloud chunk storage/dashboard read model                             |
 | `ENGRAM_PORT`                   | no                       | Runtime port (default `8080`)                                                         |
 | `ENGRAM_CLOUD_HOST`             | no                       | Bind host (default `127.0.0.1`; use `0.0.0.0` for containers)                         |
+| `ENGRAM_CLOUD_MAX_PUSH_BYTES`   | no                       | Max chunk/mutation push request body bytes (default `8388608`)                        |
 | `ENGRAM_CLOUD_ALLOWED_PROJECTS` | yes                      | Comma-separated allowlist; always required (authenticated + insecure modes)           |
 | `ENGRAM_CLOUD_TOKEN`            | yes (authenticated mode) | Enables bearer auth mode                                                              |
 | `ENGRAM_JWT_SECRET`             | yes (authenticated mode) | Must be explicitly set and non-default when token mode is enabled                     |
diff --git a/cmd/engram/cloud.go b/cmd/engram/cloud.go
@@ -112,6 +112,7 @@ var newCloudRuntime = func(cfg cloud.Config) (cloudServerRuntime, error) {
 			cloudserver.WithHost(cfg.BindHost),
 			cloudserver.WithProjectAuthorizer(projectAuth),
 			cloudserver.WithDashboardAdminToken(cfg.AdminToken),
+			cloudserver.WithMaxPushBodyBytes(cfg.MaxPushBodyBytes),
 			cloudserver.WithSyncStatusProvider(cloudDashboardStatusProvider{store: cs, projects: allowedProjects}),
 		),
 		store: cs,
diff --git a/cmd/engram/main.go b/cmd/engram/main.go
@@ -2335,6 +2335,8 @@ Environment:
   ENGRAM_DATABASE_URL
                      Postgres DSN for engram cloud serve
   ENGRAM_CLOUD_HOST  Bind host for engram cloud serve (default: 127.0.0.1)
+  ENGRAM_CLOUD_MAX_PUSH_BYTES
+                     Max cloud push payload bytes (default: 8388608)
   ENGRAM_CLOUD_TOKEN Bearer token required in authenticated cloud serve mode
   ENGRAM_CLOUD_INSECURE_NO_AUTH
                      Set to 1 ONLY for local insecure cloud serve mode (no auth)
diff --git a/cmd/engram/main_extra_test.go b/cmd/engram/main_extra_test.go
@@ -1594,6 +1594,7 @@ func TestCmdCloudServeStartsCloudRuntime(t *testing.T) {
 	t.Setenv("ENGRAM_CLOUD_HOST", "0.0.0.0")
 	t.Setenv("ENGRAM_CLOUD_TOKEN", "token-abc")
 	t.Setenv("ENGRAM_CLOUD_ALLOWED_PROJECTS", "proj-a")
+	t.Setenv("ENGRAM_CLOUD_MAX_PUSH_BYTES", "10485760")
 
 	var seen cloud.Config
 	runtimeStub := &stubCloudRuntimeServer{}
@@ -1618,6 +1619,9 @@ func TestCmdCloudServeStartsCloudRuntime(t *testing.T) {
 	if seen.BindHost != "0.0.0.0" {
 		t.Fatalf("expected cloud runtime config bind host from env, got %q", seen.BindHost)
 	}
+	if seen.MaxPushBodyBytes != 10485760 {
+		t.Fatalf("expected cloud runtime max push bytes from env, got %d", seen.MaxPushBodyBytes)
+	}
 }
 
 func TestCmdCloudServeRequiresAuthTokenByDefault(t *testing.T) {
diff --git a/cmd/engram/main_test.go b/cmd/engram/main_test.go
@@ -181,6 +181,7 @@ func TestPrintUsage(t *testing.T) {
 	for _, token := range []string{
 		"ENGRAM_DATABASE_URL",
 		"ENGRAM_CLOUD_HOST",
+		"ENGRAM_CLOUD_MAX_PUSH_BYTES",
 		"ENGRAM_CLOUD_TOKEN",
 		"ENGRAM_CLOUD_INSECURE_NO_AUTH",
 		"Cannot be combined with ENGRAM_CLOUD_TOKEN",
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
@@ -298,7 +298,7 @@ The cloud server exposes two routes registered in `cloudserver.go` and handled b
 | `POST` | `/sync/mutations/push` | Accept a batch of up to 100 mutations from the client |
 | `GET` | `/sync/mutations/pull` | Return mutations since a cursor, filtered by caller's enrolled projects |
 
-Both require `Authorization: Bearer <token>`. Push enforces the project-level sync pause (HTTP 409 on `sync_enabled=false`). Pull filters server-side.
+Both require `Authorization: Bearer <token>`. Push enforces the project-level sync pause (HTTP 409 on `sync_enabled=false`) and the configured cloud push body limit (`ENGRAM_CLOUD_MAX_PUSH_BYTES`, default 8 MiB). Pull filters server-side.
 
 ---
 
diff --git a/docs/codebase/sync-and-cloud.md b/docs/codebase/sync-and-cloud.md
@@ -59,6 +59,8 @@ Business rule: **if sync is blocked, fail loudly and visibly**. No silent drops.
 - `GET /sync/mutations/pull`
 - `/dashboard/*`
 
+`POST /sync/push` and `POST /sync/mutations/push` enforce the server-side push request body limit from `ENGRAM_CLOUD_MAX_PUSH_BYTES` (default 8 MiB).
+
 For complete route details, use [DOCS.md — HTTP API Endpoints](../../DOCS.md#http-api-endpoints).
 
 ## Cloud store: `internal/cloud/cloudstore`
diff --git a/docs/engram-cloud/quickstart.md b/docs/engram-cloud/quickstart.md
@@ -83,6 +83,9 @@ Required runtime env vars:
 - `ENGRAM_CLOUD_HOST=0.0.0.0`
 - `ENGRAM_PORT=18080`
 
+Optional runtime env vars:
+- `ENGRAM_CLOUD_MAX_PUSH_BYTES` (defaults to `8388608`)
+
 Dokploy guidance:
 1. Create a managed Postgres service.
 2. Create an app from image `ghcr.io/gentleman-programming/engram:latest`.
@@ -116,6 +119,7 @@ ENGRAM_CLOUD_ADMIN=replace-with-separate-admin-token
 ENGRAM_JWT_SECRET=replace-with-32+-byte-random-secret
 ENGRAM_CLOUD_ALLOWED_PROJECTS=engram,gentle-ai
 ENGRAM_CLOUD_HOST=0.0.0.0
+ENGRAM_CLOUD_MAX_PUSH_BYTES=8388608
 ENGRAM_PORT=18080
 ```
 
@@ -125,6 +129,7 @@ Notes:
 - `ENGRAM_CLOUD_ADMIN` is the dashboard admin token. Use a different secret from `ENGRAM_CLOUD_TOKEN`.
 - `ENGRAM_JWT_SECRET` must be an explicit, non-default strong secret in authenticated mode.
 - `ENGRAM_CLOUD_ALLOWED_PROJECTS` is required server-side and should be a comma-separated allowlist.
+- `ENGRAM_CLOUD_MAX_PUSH_BYTES` optionally raises or lowers the server-side limit for chunk and mutation push request bodies. Omit it to keep the default 8 MiB limit.
 
 Reference compose:
 
diff --git a/internal/cloud/cloudserver/cloudserver.go b/internal/cloud/cloudserver/cloudserver.go
@@ -46,19 +46,20 @@ type staticStatusProvider struct{ status dashboard.SyncStatus }
 func (s staticStatusProvider) Status() dashboard.SyncStatus { return s.status }
 
 type CloudServer struct {
-	store          ChunkStore
-	auth           Authenticator
-	projectAuth    ProjectAuthorizer
-	dashboardAdmin string
-	port           int
-	host           string
-	mux            *http.ServeMux
-	syncStatus     dashboard.SyncStatusProvider
-	listenAndServe func(addr string, handler http.Handler) error
+	store            ChunkStore
+	auth             Authenticator
+	projectAuth      ProjectAuthorizer
+	dashboardAdmin   string
+	port             int
+	host             string
+	maxPushBodyBytes int64
+	mux              *http.ServeMux
+	syncStatus       dashboard.SyncStatusProvider
+	listenAndServe   func(addr string, handler http.Handler) error
 }
 
 const defaultHost = "127.0.0.1"
-const maxPushBodyBytes int64 = 8 * 1024 * 1024
+const defaultMaxPushBodyBytes int64 = 8 * 1024 * 1024
 const maxDashboardLoginBodyBytes int64 = 16 * 1024
 const dashboardSessionCookieName = "engram_dashboard_token"
 
@@ -88,12 +89,21 @@ func WithDashboardAdminToken(adminToken string) Option {
 	}
 }
 
+func WithMaxPushBodyBytes(limit int64) Option {
+	return func(s *CloudServer) {
+		if limit > 0 {
+			s.maxPushBodyBytes = limit
+		}
+	}
+}
+
 func New(store ChunkStore, authSvc Authenticator, port int, opts ...Option) *CloudServer {
 	s := &CloudServer{
-		store: store,
-		auth:  authSvc,
-		port:  port,
-		host:  defaultHost,
+		store:            store,
+		auth:             authSvc,
+		port:             port,
+		host:             defaultHost,
+		maxPushBodyBytes: defaultMaxPushBodyBytes,
 		syncStatus: staticStatusProvider{status: dashboard.SyncStatus{
 			Phase:         "degraded",
 			ReasonCode:    constants.ReasonTransportFailed,
@@ -128,6 +138,13 @@ func (s *CloudServer) Handler() http.Handler {
 	return s.mux
 }
 
+func (s *CloudServer) pushBodyLimit() int64 {
+	if s.maxPushBodyBytes > 0 {
+		return s.maxPushBodyBytes
+	}
+	return defaultMaxPushBodyBytes
+}
+
 func (s *CloudServer) routes() {
 	s.mux = http.NewServeMux()
 	s.mux.HandleFunc("GET /health", s.handleHealth)
@@ -346,6 +363,7 @@ func (s *CloudServer) handlePullChunk(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *CloudServer) handlePushChunk(w http.ResponseWriter, r *http.Request) {
+	maxPushBodyBytes := s.pushBodyLimit()
 	r.Body = http.MaxBytesReader(w, r.Body, maxPushBodyBytes)
 	var req struct {
 		ChunkID         string          `json:"chunk_id"`
diff --git a/internal/cloud/cloudserver/cloudserver_test.go b/internal/cloud/cloudserver/cloudserver_test.go
@@ -775,16 +775,16 @@ func TestHandlerProjectScopeForbiddenReturnsPolicyClassPayload(t *testing.T) {
 }
 
 func TestHandlerPushRejectsOversizedPayload(t *testing.T) {
-	srv := New(&fakeStore{}, fakeAuth{}, 0)
-	tooLarge := strings.Repeat("x", int(maxPushBodyBytes)+1)
+	srv := New(&fakeStore{}, fakeAuth{}, 0, WithMaxPushBodyBytes(128))
+	tooLarge := strings.Repeat("x", 129)
 	body := bytes.NewBufferString(`{"project":"proj-a","created_by":"tester","data":"` + tooLarge + `"}`)
 	rec := httptest.NewRecorder()
 	srv.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/sync/push", body))
 	if rec.Code != http.StatusRequestEntityTooLarge {
 		t.Fatalf("expected 413, got %d body=%q", rec.Code, rec.Body.String())
 	}
-	if !strings.Contains(rec.Body.String(), "payload too large") {
-		t.Fatalf("expected clear oversized payload error, got body=%q", rec.Body.String())
+	if !strings.Contains(rec.Body.String(), "payload too large") || !strings.Contains(rec.Body.String(), "max 128 bytes") {
+		t.Fatalf("expected clear oversized payload error with configured limit, got body=%q", rec.Body.String())
 	}
 }
 
diff --git a/internal/cloud/cloudserver/mutations.go b/internal/cloud/cloudserver/mutations.go
@@ -3,6 +3,7 @@ package cloudserver
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
 	"net/http"
@@ -57,14 +58,20 @@ const defaultPullLimit = 100
 // ─── Handlers ────────────────────────────────────────────────────────────────
 
 // handleMutationPush handles POST /sync/mutations/push.
-// REQ-200: bearer auth, body limit 8 MiB, batch size cap 100, pause gate (409 on sync_enabled=false).
+// REQ-200: bearer auth, configurable body limit defaulting to 8 MiB, batch size cap 100, pause gate (409 on sync_enabled=false).
 // BC2: project authorization is enforced for every distinct project in the batch.
 // BW9: 409 pause response uses writeActionableError for structured error envelope.
 func (s *CloudServer) handleMutationPush(w http.ResponseWriter, r *http.Request) {
+	maxPushBodyBytes := s.pushBodyLimit()
 	r.Body = http.MaxBytesReader(w, r.Body, maxPushBodyBytes)
 
 	var req mutationPushEnvelope
 	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		var maxBytesErr *http.MaxBytesError
+		if errors.As(err, &maxBytesErr) {
+			writeActionableError(w, http.StatusRequestEntityTooLarge, constants.UpgradeErrorClassRepairable, constants.UpgradeErrorCodePayloadTooLarge, fmt.Sprintf("push payload too large (max %d bytes)", maxPushBodyBytes))
+			return
+		}
 		http.Error(w, fmt.Sprintf("invalid request body: %v", err), http.StatusBadRequest)
 		return
 	}
diff --git a/internal/cloud/cloudserver/mutations_test.go b/internal/cloud/cloudserver/mutations_test.go
@@ -1395,6 +1395,26 @@ func TestMutationPushPausedWithCreatedBy(t *testing.T) {
 	}
 }
 
+func TestMutationPushRejectsOversizedPayload(t *testing.T) {
+	ms := newFakeMutationStore()
+	srv := New(ms, multiProjectAuth{token: "secret", projects: []string{"proj-a"}}, 0, WithMaxPushBodyBytes(128))
+	tooLarge := strings.Repeat("x", 129)
+	body := bytes.NewBufferString(`{"entries":[{"project":"proj-a","entity":"observation","entity_key":"obs-1","op":"upsert","payload":"` + tooLarge + `"}]}`)
+
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodPost, "/sync/mutations/push", body)
+	req.Header.Set("Authorization", "Bearer secret")
+	req.Header.Set("Content-Type", "application/json")
+	srv.Handler().ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusRequestEntityTooLarge {
+		t.Fatalf("expected 413, got %d body=%q", rec.Code, rec.Body.String())
+	}
+	if !strings.Contains(rec.Body.String(), "payload too large") || !strings.Contains(rec.Body.String(), "max 128 bytes") {
+		t.Fatalf("expected clear oversized payload error with configured limit, got body=%q", rec.Body.String())
+	}
+}
+
 // TestMutationPushPausedWithoutCreatedByDefaultsUnknown verifies that missing
 // created_by defaults contributor to "unknown". REQ-406 scenario 2, 2.1.4.
 func TestMutationPushPausedWithoutCreatedByDefaultsUnknown(t *testing.T) {
diff --git a/internal/cloud/config.go b/internal/cloud/config.go
@@ -7,26 +7,29 @@ import (
 )
 
 type Config struct {
-	DSN             string
-	JWTSecret       string
-	CORSOrigins     []string
-	MaxPool         int
-	Port            int
-	BindHost        string
-	AdminToken      string
-	AllowedProjects []string
+	DSN              string
+	JWTSecret        string
+	CORSOrigins      []string
+	MaxPool          int
+	Port             int
+	BindHost         string
+	AdminToken       string
+	AllowedProjects  []string
+	MaxPushBodyBytes int64
 }
 
 const DefaultJWTSecret = "engram-dev-jwt-secret-for-local-smoke-1234"
+const DefaultMaxPushBodyBytes int64 = 8 * 1024 * 1024
 
 func DefaultConfig() Config {
 	return Config{
-		DSN:         "postgres://engram:engram_dev@localhost:5433/engram_cloud?sslmode=disable",
-		JWTSecret:   DefaultJWTSecret,
-		CORSOrigins: []string{"*"},
-		MaxPool:     10,
-		Port:        8080,
-		BindHost:    "127.0.0.1",
+		DSN:              "postgres://engram:engram_dev@localhost:5433/engram_cloud?sslmode=disable",
+		JWTSecret:        DefaultJWTSecret,
+		CORSOrigins:      []string{"*"},
+		MaxPool:          10,
+		Port:             8080,
+		BindHost:         "127.0.0.1",
+		MaxPushBodyBytes: DefaultMaxPushBodyBytes,
 	}
 }
 
@@ -53,6 +56,11 @@ func ConfigFromEnv() Config {
 	if v := strings.TrimSpace(os.Getenv("ENGRAM_CLOUD_HOST")); v != "" {
 		cfg.BindHost = v
 	}
+	if v := strings.TrimSpace(os.Getenv("ENGRAM_CLOUD_MAX_PUSH_BYTES")); v != "" {
+		if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
+			cfg.MaxPushBodyBytes = n
+		}
+	}
 	if v := strings.TrimSpace(os.Getenv("ENGRAM_CLOUD_ALLOWED_PROJECTS")); v != "" {
 		parts := strings.Split(v, ",")
 		projects := make([]string, 0, len(parts))
diff --git a/internal/cloud/config_test.go b/internal/cloud/config_test.go
@@ -31,6 +31,34 @@ func TestConfigFromEnvAllowedProjects(t *testing.T) {
 	}
 }
 
+func TestConfigFromEnvMaxPushBodyBytes(t *testing.T) {
+	t.Run("default is 8 MiB", func(t *testing.T) {
+		t.Setenv("ENGRAM_CLOUD_MAX_PUSH_BYTES", "")
+		cfg := ConfigFromEnv()
+		if cfg.MaxPushBodyBytes != DefaultMaxPushBodyBytes {
+			t.Fatalf("expected default max push bytes %d, got %d", DefaultMaxPushBodyBytes, cfg.MaxPushBodyBytes)
+		}
+	})
+
+	t.Run("env overrides with positive integer", func(t *testing.T) {
+		t.Setenv("ENGRAM_CLOUD_MAX_PUSH_BYTES", "10485760")
+		cfg := ConfigFromEnv()
+		if cfg.MaxPushBodyBytes != 10485760 {
+			t.Fatalf("expected max push bytes override 10485760, got %d", cfg.MaxPushBodyBytes)
+		}
+	})
+
+	for _, value := range []string{"0", "-1", "not-a-number"} {
+		t.Run("invalid value keeps default "+value, func(t *testing.T) {
+			t.Setenv("ENGRAM_CLOUD_MAX_PUSH_BYTES", value)
+			cfg := ConfigFromEnv()
+			if cfg.MaxPushBodyBytes != DefaultMaxPushBodyBytes {
+				t.Fatalf("expected default max push bytes for %q, got %d", value, cfg.MaxPushBodyBytes)
+			}
+		})
+	}
+}
+
 func TestIsDefaultJWTSecret(t *testing.T) {
 	t.Run("default secret returns true", func(t *testing.T) {
 		if !IsDefaultJWTSecret(DefaultJWTSecret) {
