diff --git a/cli/daemon/run/infra/infra.go b/cli/daemon/run/infra/infra.go index 5f02ee11e3..1a12fb67f3 100644 --- a/cli/daemon/run/infra/infra.go +++ b/cli/daemon/run/infra/infra.go @@ -3,6 +3,8 @@ package infra import ( "context" "fmt" + "math" + "os" "strconv" "sync" "time" @@ -402,6 +404,38 @@ func (rm *ResourceManager) SQLDatabaseConfig(db *meta.SQLDatabase) (config.SQLDa return dbCfg, nil } +// defaultSQLPoolBudget is the total pgx connection budget shared across +// the databases in a locally-managed Postgres cluster. It leaves headroom +// below Postgres's default server-side max_connections of 100 for admin +// and replication slots. +const defaultSQLPoolBudget = 96 + +// SQLDatabaseMaxConnections returns the per-database pgx MaxConns to use +// for a locally-managed Postgres cluster hosting numLocalDBs databases. +// +// Reads ENCORE_SQLDB_POOL_BUDGET to override the total budget; falls back +// to defaultSQLPoolBudget. The env var is read on every call, but the +// daemon's environment is frozen at daemon startup — so setting the var +// in the shell that spawns the daemon is what matters; changing it +// afterwards requires restarting the daemon to take effect. +// +// The result is clamped to int32 to match the SQLConnectionPool proto +// field on the consuming side. +func (rm *ResourceManager) SQLDatabaseMaxConnections(numLocalDBs int) int { + if numLocalDBs <= 0 { + return 0 + } + budget := defaultSQLPoolBudget + if v := os.Getenv("ENCORE_SQLDB_POOL_BUDGET"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + budget = n + } else { + rm.log.Warn().Str("value", v).Msg("ignoring invalid ENCORE_SQLDB_POOL_BUDGET; expected positive integer") + } + } + return min(max(budget/numLocalDBs, 1), math.MaxInt32) +} + // PubSubProviderConfig returns the PubSub provider configuration. func (rm *ResourceManager) PubSubProviderConfig() (config.PubsubProvider, error) { nsq := rm.GetPubSub() diff --git a/cli/daemon/run/infra/infra_test.go b/cli/daemon/run/infra/infra_test.go new file mode 100644 index 0000000000..ac39bd5e27 --- /dev/null +++ b/cli/daemon/run/infra/infra_test.go @@ -0,0 +1,43 @@ +package infra + +import ( + "math" + "strconv" + "testing" + + "github.com/rs/zerolog" +) + +func TestSQLDatabaseMaxConnections(t *testing.T) { + rm := &ResourceManager{log: zerolog.Nop()} + + tests := []struct { + name string + envVar string + numDBs int + wantVal int + }{ + {name: "default budget, one db", numDBs: 1, wantVal: 96}, + {name: "default budget, three dbs", numDBs: 3, wantVal: 32}, + {name: "default budget, six dbs", numDBs: 6, wantVal: 16}, + {name: "default budget, zero dbs returns zero", numDBs: 0, wantVal: 0}, + {name: "default budget, negative dbs returns zero", numDBs: -1, wantVal: 0}, + {name: "default budget floors at 1 when many dbs", numDBs: 200, wantVal: 1}, + {name: "custom budget", envVar: "400", numDBs: 10, wantVal: 40}, + {name: "custom budget floors at 1", envVar: "2", numDBs: 10, wantVal: 1}, + {name: "invalid env var falls back to default", envVar: "not-a-number", numDBs: 4, wantVal: 24}, + {name: "zero env var falls back to default", envVar: "0", numDBs: 4, wantVal: 24}, + {name: "negative env var falls back to default", envVar: "-5", numDBs: 4, wantVal: 24}, + {name: "budget larger than int32 max clamps to int32 max", envVar: strconv.Itoa(math.MaxInt32 + 1000), numDBs: 1, wantVal: math.MaxInt32}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("ENCORE_SQLDB_POOL_BUDGET", tc.envVar) + if got := rm.SQLDatabaseMaxConnections(tc.numDBs); got != tc.wantVal { + t.Fatalf("SQLDatabaseMaxConnections(%d) with env=%q = %d, want %d", + tc.numDBs, tc.envVar, got, tc.wantVal) + } + }) + } +} diff --git a/cli/daemon/run/runtime_config2.go b/cli/daemon/run/runtime_config2.go index ba30109153..5ee1d5d3ed 100644 --- a/cli/daemon/run/runtime_config2.go +++ b/cli/daemon/run/runtime_config2.go @@ -63,6 +63,7 @@ type RuntimeConfigGenerator struct { PubSubProviderConfig() (config.PubsubProvider, error) SQLDatabaseConfig(db *meta.SQLDatabase) (config.SQLDatabase, error) + SQLDatabaseMaxConnections(numLocalDBs int) int PubSubTopicConfig(topic *meta.PubSubTopic) (config.PubsubProvider, config.PubsubTopic, error) PubSubSubscriptionConfig(topic *meta.PubSubTopic, sub *meta.PubSubTopic_Subscription) (config.PubsubSubscription, error) RedisConfig(redis *meta.CacheCluster) (config.RedisServer, config.RedisDatabase, error) @@ -306,6 +307,18 @@ func (g *RuntimeConfigGenerator) initialize() error { TlsConfig: tlsConfig, }) + // Count databases hosted by the locally-managed cluster so the + // shared pgx connection budget (see infra.SQLDatabaseMaxConnections) + // can be divided evenly among them. External databases connect to + // their own Postgres servers and aren't subject to the local budget. + numLocalDBs := 0 + for _, db := range g.md.SqlDatabases { + if _, external := g.DefinedSecrets["sqldb::"+db.Name]; !external { + numLocalDBs++ + } + } + localMaxConns := g.infraManager.SQLDatabaseMaxConnections(numLocalDBs) + for _, db := range g.md.SqlDatabases { if externalDB, ok := g.DefinedSecrets["sqldb::"+db.Name]; ok { var extCfg struct { @@ -362,6 +375,10 @@ func (g *RuntimeConfigGenerator) initialize() error { Password: toSecret([]byte(dbConfig.Password)), ClientCertRid: nil, }) + maxConns := int32(dbConfig.MaxConnections) + if maxConns == 0 { + maxConns = int32(localMaxConns) + } cluster.SQLDatabase(&runtimev1.SQLDatabase{ Rid: newRid(), EncoreName: dbConfig.EncoreName, @@ -371,7 +388,7 @@ func (g *RuntimeConfigGenerator) initialize() error { IsReadonly: false, RoleRid: roleRid, MinConnections: int32(dbConfig.MinConnections), - MaxConnections: int32(dbConfig.MaxConnections), + MaxConnections: maxConns, }) } diff --git a/cli/daemon/sqldb/docker/docker.go b/cli/daemon/sqldb/docker/docker.go index 633799cdb4..cb78f0f793 100644 --- a/cli/daemon/sqldb/docker/docker.go +++ b/cli/daemon/sqldb/docker/docker.go @@ -8,6 +8,7 @@ import ( "io" "os" "os/exec" + "strconv" "strings" "time" @@ -156,6 +157,20 @@ func (d *Driver) CreateCluster(ctx context.Context, p *sqldb.CreateParams, log z Image) } + // Allow CI / power users to raise the Postgres server's max_connections + // ceiling above its default (100). Only applied when creating a fresh + // container — the daemon is long-lived and `docker start` on an existing + // container reuses its original args. To pick up a new value locally, + // stop the daemon and `docker rm` the encore-pg container so the next + // run creates a fresh one. + if v := os.Getenv("ENCORE_SQLDB_MAX_CONNECTIONS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + args = append(args, "-c", "max_connections="+strconv.Itoa(n)) + } else { + log.Warn().Str("value", v).Msg("ignoring invalid ENCORE_SQLDB_MAX_CONNECTIONS; expected positive integer") + } + } + cmd := exec.CommandContext(ctx, "docker", args...) if out, err := cmd.CombinedOutput(); err != nil { return nil, errors.Wrapf(err, "could not start sql database as docker container: %s", out)