Skip to content

Commit b67c735

Browse files
committed
Add ContainerOptions.allowPrivileged for FUSE in local-dev Docker containers
Containers using FUSE work in production but break in `wrangler dev` because workerd doesn't grant CAP_SYS_ADMIN, mount /dev/fuse, or relax AppArmor on the container it creates. Without those three knobs the mount("fuse", ...) syscall inside the container fails. Add a new boolean to ContainerOptions in workerd.capnp, off by default: struct ContainerOptions { imageName @0 :Text; allowPrivileged @1 :Bool = false; # NEW } When set on a DurableObjectNamespace's container config, ContainerClient adds the minimum HostConfig fields needed for FUSE to the POST /containers/create payload: - CapAdd: ["SYS_ADMIN"] (for the mount() syscall) - Devices: [/dev/fuse] (so the kernel has something to open) - SecurityOpt: ["apparmor:unconfined"] (bypasses the default docker apparmor profile's mount block) No Privileged=true. Mirrors what Cloudflare's production container runtime effectively provides for FUSE workers, without granting all Linux capabilities or full /dev passthrough. server.c++ reads the flag from the per-DO ContainerOptions config and threads it through to the ContainerClient constructor. Off by default means existing configs are unaffected. Only the local Docker code path is touched; production is unaffected. Also adds missing $Json.name annotations to docker_api::DeviceMapping fields in docker-api.capnp so they serialize as PathOnHost / PathInContainer / CgroupPermissions to match the Docker API. This is load-bearing for the Devices bind above — without it, the entry serializes as camelCase and Docker silently ignores it. Before this PR no call site populated Devices, so the missing annotations were latent. Follows the existing pattern of createSidecarContainer() which already sets CapAdd=[NET_ADMIN] for its own need at container-client.c++:1958. The matching workers-sdk PR (cloudflare/workers-sdk#13748) adds `dev.enable_containers_privileged_mode` and `--enable-containers-privileged-mode` so wrangler / vite-plugin users can opt in. End-to-end reproduction (libc mount("fuse", ...) syscall succeeding inside the container with the flag set, and failing without it) at https://github.com/Ben2W/workerd-fuse-local-repro Signed-off-by: Ben Werner <bewerner23@gmail.com>
1 parent 4d9a037 commit b67c735

6 files changed

Lines changed: 100 additions & 8 deletions

File tree

src/workerd/server/container-client-test.c++

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,5 +195,58 @@ KJ_TEST("ContainerCreateRequest encodes HostConfig Dns") {
195195
KJ_EXPECT(decodedDns[1] == "8.8.8.8");
196196
}
197197

198+
KJ_TEST("ContainerCreateRequest encodes privileged FUSE HostConfig") {
199+
capnp::JsonCodec codec;
200+
codec.handleByAnnotation<docker_api::Docker::ContainerCreateRequest>();
201+
202+
capnp::MallocMessageBuilder message;
203+
auto root = message.initRoot<docker_api::Docker::ContainerCreateRequest>();
204+
root.setImage("test-image");
205+
206+
auto hostConfig = root.initHostConfig();
207+
208+
auto capAdd = hostConfig.initCapAdd(1);
209+
capAdd.set(0, "SYS_ADMIN");
210+
211+
auto devices = hostConfig.initDevices(1);
212+
auto fuseDev = devices[0];
213+
fuseDev.setPathOnHost("/dev/fuse");
214+
fuseDev.setPathInContainer("/dev/fuse");
215+
fuseDev.setCgroupPermissions("rwm");
216+
217+
auto securityOpt = hostConfig.initSecurityOpt(1);
218+
securityOpt.set(0, "apparmor:unconfined");
219+
220+
auto json = codec.encode(root);
221+
auto jsonText = json.asPtr();
222+
223+
KJ_EXPECT(jsonText.contains("\"CapAdd\""));
224+
KJ_EXPECT(jsonText.contains("\"SYS_ADMIN\""));
225+
KJ_EXPECT(jsonText.contains("\"Devices\""));
226+
KJ_EXPECT(jsonText.contains("\"PathOnHost\":\"/dev/fuse\""));
227+
KJ_EXPECT(jsonText.contains("\"PathInContainer\":\"/dev/fuse\""));
228+
KJ_EXPECT(jsonText.contains("\"CgroupPermissions\":\"rwm\""));
229+
KJ_EXPECT(jsonText.contains("\"SecurityOpt\""));
230+
KJ_EXPECT(jsonText.contains("\"apparmor:unconfined\""));
231+
232+
auto decoded = decodeJsonResponse<docker_api::Docker::ContainerCreateRequest>(jsonText);
233+
auto decodedRoot = decoded->getRoot<docker_api::Docker::ContainerCreateRequest>();
234+
auto decodedHostConfig = decodedRoot.getHostConfig();
235+
auto decodedCapAdd = decodedHostConfig.getCapAdd();
236+
auto decodedDevices = decodedHostConfig.getDevices();
237+
auto decodedSecurityOpt = decodedHostConfig.getSecurityOpt();
238+
239+
KJ_REQUIRE(decodedCapAdd.size() == 1);
240+
KJ_EXPECT(decodedCapAdd[0] == "SYS_ADMIN");
241+
242+
KJ_REQUIRE(decodedDevices.size() == 1);
243+
KJ_EXPECT(decodedDevices[0].getPathOnHost() == "/dev/fuse");
244+
KJ_EXPECT(decodedDevices[0].getPathInContainer() == "/dev/fuse");
245+
KJ_EXPECT(decodedDevices[0].getCgroupPermissions() == "rwm");
246+
247+
KJ_REQUIRE(decodedSecurityOpt.size() == 1);
248+
KJ_EXPECT(decodedSecurityOpt[0] == "apparmor:unconfined");
249+
}
250+
198251
} // namespace
199252
} // namespace workerd::server

src/workerd/server/container-client.c++

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -939,7 +939,8 @@ ContainerClient::ContainerClient(capnp::ByteStreamFactory& byteStreamFactory,
939939
kj::TaskSet& waitUntilTasks,
940940
kj::Promise<void> pendingCleanup,
941941
kj::Function<void(kj::Promise<void>)> cleanupCallback,
942-
ChannelTokenHandler& channelTokenHandler)
942+
ChannelTokenHandler& channelTokenHandler,
943+
bool allowPrivileged)
943944
: byteStreamFactory(byteStreamFactory),
944945
timer(timer),
945946
network(network),
@@ -948,6 +949,7 @@ ContainerClient::ContainerClient(capnp::ByteStreamFactory& byteStreamFactory,
948949
sidecarContainerName(kj::encodeUriComponent(kj::str(containerName, "-proxy"))),
949950
imageName(kj::mv(imageName)),
950951
containerEgressInterceptorImage(kj::mv(containerEgressInterceptorImage)),
952+
allowPrivileged(allowPrivileged),
951953
waitUntilTasks(waitUntilTasks),
952954
pendingCleanup(kj::mv(pendingCleanup).fork()),
953955
cleanupCallback(kj::mv(cleanupCallback)),
@@ -1722,6 +1724,27 @@ kj::Promise<void> ContainerClient::createContainer(kj::StringPtr effectiveImage,
17221724
}
17231725
}
17241726

1727+
// Docker doesn't grant FUSE access by default — when the user opts in via
1728+
// ContainerOptions.allowPrivileged, enable the minimum permissions for it.
1729+
if (allowPrivileged) {
1730+
{
1731+
auto capAdd = hostConfig.initCapAdd(1);
1732+
capAdd.set(0, "SYS_ADMIN");
1733+
}
1734+
{
1735+
auto devices = hostConfig.initDevices(1);
1736+
auto fuseDev = devices[0];
1737+
fuseDev.setPathOnHost("/dev/fuse");
1738+
fuseDev.setPathInContainer("/dev/fuse");
1739+
fuseDev.setCgroupPermissions("rwm");
1740+
}
1741+
{
1742+
// Linux-only: no-op on hosts without AppArmor (e.g. macOS).
1743+
auto securityOpt = hostConfig.initSecurityOpt(1);
1744+
securityOpt.set(0, "apparmor:unconfined");
1745+
}
1746+
}
1747+
17251748
auto response = co_await dockerApiRequest(network, kj::str(dockerPath), kj::HttpMethod::POST,
17261749
kj::str("/containers/create?name=", containerName), codec.encode(jsonRoot));
17271750

src/workerd/server/container-client.h

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,8 @@ class ContainerClient final: public rpc::Container::Server, public kj::Refcounte
6868
kj::TaskSet& waitUntilTasks,
6969
kj::Promise<void> pendingCleanup,
7070
kj::Function<void(kj::Promise<void>)> cleanupCallback,
71-
ChannelTokenHandler& channelTokenHandler);
71+
ChannelTokenHandler& channelTokenHandler,
72+
bool allowPrivileged);
7273

7374
~ContainerClient() noexcept(false);
7475

@@ -104,6 +105,11 @@ class ContainerClient final: public rpc::Container::Server, public kj::Refcounte
104105
// Container egress interceptor image name (sidecar for egress proxy)
105106
kj::String containerEgressInterceptorImage;
106107

108+
// When true, createContainer() injects the HostConfig fields needed for FUSE
109+
// (CAP_SYS_ADMIN, /dev/fuse, AppArmor unconfined). See ContainerOptions in
110+
// workerd.capnp.
111+
bool allowPrivileged;
112+
107113
kj::TaskSet& waitUntilTasks;
108114

109115
// Forked promise representing pending cleanup from a previous ContainerClient for the same

src/workerd/server/docker-api.capnp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,9 @@ struct Docker {
3232
}
3333

3434
struct DeviceMapping {
35-
pathOnHost @0 :Text;
36-
pathInContainer @1 :Text;
37-
cgroupPermissions @2 :Text;
35+
pathOnHost @0 :Text $Json.name("PathOnHost");
36+
pathInContainer @1 :Text $Json.name("PathInContainer");
37+
cgroupPermissions @2 :Text $Json.name("CgroupPermissions");
3838
}
3939

4040
struct DeviceRequest {

src/workerd/server/server.c++

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2936,6 +2936,7 @@ class Server::WorkerService final: public Service,
29362936
KJ_IF_SOME(config, containerOptions) {
29372937
KJ_ASSERT(config.hasImageName(), "Image name is required");
29382938
auto imageName = config.getImageName();
2939+
bool allowPrivileged = config.getAllowPrivileged();
29392940
kj::String containerId;
29402941
KJ_SWITCH_ONEOF(id) {
29412942
KJ_CASE_ONEOF(globalId, kj::Own<ActorIdFactory::ActorId>) {
@@ -2947,7 +2948,8 @@ class Server::WorkerService final: public Service,
29472948
}
29482949

29492950
container = ns.getContainerClient(
2950-
kj::str("workerd-", KJ_ASSERT_NONNULL(uniqueKey), "-", containerId), imageName);
2951+
kj::str("workerd-", KJ_ASSERT_NONNULL(uniqueKey), "-", containerId), imageName,
2952+
allowPrivileged);
29512953
}
29522954

29532955
auto actor = actorClass->newActor(getTracker(), Worker::Actor::cloneId(id),
@@ -2992,7 +2994,7 @@ class Server::WorkerService final: public Service,
29922994
}
29932995

29942996
kj::Own<ContainerClient> getContainerClient(
2995-
kj::StringPtr containerId, kj::StringPtr imageName) {
2997+
kj::StringPtr containerId, kj::StringPtr imageName, bool allowPrivileged) {
29962998
KJ_IF_SOME(existingClient, containerClients.find(containerId)) {
29972999
return existingClient->addRef();
29983000
}
@@ -3050,7 +3052,8 @@ class Server::WorkerService final: public Service,
30503052
kj::str(dockerPathRef), kj::str(containerId), kj::str(imageName),
30513053
kj::str(KJ_ASSERT_NONNULL(containerEgressInterceptorImage,
30523054
"containerEgressInterceptorImage must be configured for containers.")),
3053-
waitUntilTasks, kj::mv(previousCleanup), kj::mv(cleanupCallback), channelTokenHandler);
3055+
waitUntilTasks, kj::mv(previousCleanup), kj::mv(cleanupCallback), channelTokenHandler,
3056+
allowPrivileged);
30543057

30553058
// Store raw pointer in map (does not own)
30563059
containerClients.insert(kj::str(containerId), client.get());

src/workerd/server/workerd.capnp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -673,6 +673,13 @@ struct Worker {
673673
imageName @0 :Text;
674674
# Image name to be used to create the container using supported provider.
675675
# By default, we pull the "latest" tag of this image.
676+
677+
allowPrivileged @1 :Bool = false;
678+
# When true, workerd creates the container with the elevated permissions
679+
# needed for FUSE in local dev: CAP_SYS_ADMIN, /dev/fuse passthrough, and
680+
# AppArmor unconfined. Off by default — only set this from a dev-only
681+
# config path. Has no effect on production Cloudflare Containers, which
682+
# use a different runtime path.
676683
}
677684
}
678685

0 commit comments

Comments
 (0)