feat(docker,podman): bump RLIMIT_NOFILE to kernel max for every container (#225)

## Summary

EL clients keep many file descriptors open while indexing chain DBs, and
typical container defaults (1024 soft / 4096 hard) trip `too many open
files` errors on long benchmark runs. Lift `RLIMIT_NOFILE` to the
highest value the host kernel allows, applied uniformly to every
container the runner creates — both Docker and Podman runtimes.

### Files

**New**
- `pkg/docker/ulimit.go` — `HostMaxNofile()` reads
`/proc/sys/fs/nr_open` (the kernel-wide ceiling above which
`RLIMIT_NOFILE` cannot be raised). Falls back to `DefaultMaxNofile =
1048576` when the file is missing/empty/unparseable (non-Linux hosts,
restricted procfs, etc.).
- `pkg/docker/ulimit_test.go` — non-zero check + Linux
`/proc/sys/fs/nr_open` sanity check.

**Updated**
- `pkg/docker/docker.go` — `hostCfg.Ulimits = []*container.Ulimit{{Name:
"nofile", Hard: nofile, Soft: nofile}}` with `nofile =
int64(HostMaxNofile())`. Set after the literal because `Ulimits` lives
on the embedded `Resources` struct.
- `pkg/podman/podman.go` — `s.Rlimits = append(s.Rlimits,
specs.POSIXRlimit{Type: "RLIMIT_NOFILE", Hard: nofile, Soft: nofile})`
with `nofile = docker.HostMaxNofile()`. Podman's specgen takes `uint64`
directly.

No config knob — always-on by design. Both `CreateContainer` paths
(regular run + init container) get the bump.

### Tests

- [x] `go build ./...` clean
- [x] `go test ./pkg/docker/...` passes
- [x] `golangci-lint run --new-from-rev=origin/master` clean

## Test plan

- [x] `docker inspect <bench-container>` should show `HostConfig.Ulimits
= [{"Name":"nofile","Soft":1048576,"Hard":1048576}]` (or whatever your
host's `cat /proc/sys/fs/nr_open` is).
- [x] Same for `podman inspect <bench-container>` under
`.HostConfig.Ulimits`.
- [x] Run a long benchmark (e.g. EEST stateful suite) on a client
previously hitting EMFILE; confirm the failure no longer reproduces.

Author: skylenet

pkg/docker/docker.go

@@ -168,6 +168,9 @@ func (m *manager) Start(ctx context.Context) error {
 	}
 
 	m.log.Debug("Connected to Docker daemon")
+	m.log.WithField("nofile", HostMaxNofile()).Info(
+		"Container RLIMIT_NOFILE bumped to host kernel max",
+	)
 
 	return nil
 }
@@ -253,12 +256,21 @@ func (m *manager) CreateContainer(ctx context.Context, spec *ContainerSpec) (str
 		Cmd:        spec.Command,
 	}
 
+	// Bump RLIMIT_NOFILE to the kernel's hard ceiling so EL clients
+	// don't trip "too many open files" errors during long benchmark
+	// runs. Applied to every container we create. Ulimits lives on the
+	// embedded Resources struct, so it has to be set after the literal.
+	nofile := int64(HostMaxNofile()) //nolint:gosec // bounded by kernel nr_open, fits in int64.
+
 	hostCfg := &container.HostConfig{
 		Mounts:      mounts,
 		NetworkMode: container.NetworkMode(spec.NetworkName),
 		CapAdd:      spec.CapAdd,
 		SecurityOpt: spec.SecurityOpt,
 	}
+	hostCfg.Ulimits = []*container.Ulimit{
+		{Name: "nofile", Hard: nofile, Soft: nofile},
+	}
 
 	// Apply resource limits if configured.
 	if spec.ResourceLimits != nil {

pkg/docker/ulimit.go

@@ -0,0 +1,34 @@
+package docker
+
+import (
+	"os"
+	"strconv"
+	"strings"
+)
+
+// DefaultMaxNofile is the fallback nofile (open-file-descriptor) limit
+// applied to containers when /proc/sys/fs/nr_open cannot be read (e.g.
+// non-Linux hosts). Matches the common Linux kernel default cap.
+const DefaultMaxNofile uint64 = 1048576
+
+// HostMaxNofile returns the highest RLIMIT_NOFILE value that containers
+// on this host are allowed to set. On Linux it reads
+// /proc/sys/fs/nr_open, the kernel-wide cap above which RLIMIT_NOFILE
+// cannot be raised. On any other platform — or if the file is missing,
+// empty, or unparsable — it returns DefaultMaxNofile. EL clients tend
+// to keep many file descriptors open; bumping nofile to the kernel's
+// hard ceiling avoids spurious "too many open files" failures during
+// long benchmark runs.
+func HostMaxNofile() uint64 {
+	data, err := os.ReadFile("/proc/sys/fs/nr_open")
+	if err != nil {
+		return DefaultMaxNofile
+	}
+
+	v, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64)
+	if err != nil || v == 0 {
+		return DefaultMaxNofile
+	}
+
+	return v
+}

pkg/docker/ulimit_test.go

@@ -0,0 +1,31 @@
+package docker
+
+import (
+	"os"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHostMaxNofile_NonZero(t *testing.T) {
+	got := HostMaxNofile()
+	assert.NotZero(t, got, "HostMaxNofile must always return a non-zero value")
+}
+
+func TestHostMaxNofile_LinuxReadsProc(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("Linux-only test: /proc/sys/fs/nr_open")
+	}
+
+	data, err := os.ReadFile("/proc/sys/fs/nr_open")
+	if err != nil {
+		t.Skipf("could not read /proc/sys/fs/nr_open: %v", err)
+	}
+
+	require.NotEmpty(t, data)
+
+	got := HostMaxNofile()
+	assert.GreaterOrEqual(t, got, uint64(1024), "kernel nr_open should be at least 1024")
+}

pkg/podman/podman.go

@@ -115,6 +115,10 @@ func (m *manager) Start(ctx context.Context) error {
 		"runtime": info.Host.OCIRuntime.Name,
 	}).Debug("Connected to Podman daemon")
 
+	m.log.WithField("nofile", docker.HostMaxNofile()).Info(
+		"Container RLIMIT_NOFILE bumped to host kernel max",
+	)
+
 	return nil
 }
 
@@ -248,6 +252,16 @@ func (m *manager) CreateContainer(
 		}
 	}
 
+	// Bump RLIMIT_NOFILE to the kernel's hard ceiling so EL clients
+	// don't trip "too many open files" errors during long benchmark
+	// runs. Applied to every container we create.
+	nofile := docker.HostMaxNofile()
+	s.Rlimits = append(s.Rlimits, specs.POSIXRlimit{
+		Type: "RLIMIT_NOFILE",
+		Hard: nofile,
+		Soft: nofile,
+	})
+
 	// Apply resource limits.
 	if spec.ResourceLimits != nil {
 		s.ResourceLimits = &specs.LinuxResources{}