Skip to content

Commit 00d1f8d

Browse files
mmabroukmmabrouk
committed
fix(hosting): bundle the SeaweedFS store in the gh.local and gh.ssl compose variants
PR #5313 added the bundled store and flipped the gh env default to http://seaweedfs:8333, but only added the service to docker-compose.gh.yml. The gh.local (build-from-source) and gh.ssl variants reuse the same gh env file, so they inherited the endpoint default with no such service on the network. Add the identical seaweedfs service, api depends_on, and seaweed-data volume to oss gh.local, oss gh.ssl, and ee gh.local, plus a multi-replica JWT note in the gh env examples. Claude-Session: https://claude.ai/code/session_01XhENr63WL9npkKrJGnzDc1
1 parent 5605cc2 commit 00d1f8d

5 files changed

Lines changed: 192 additions & 0 deletions

File tree

‎hosting/docker-compose/ee/docker-compose.gh.local.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ services:
6868
condition: service_healthy
6969
redis-durable:
7070
condition: service_healthy
71+
seaweedfs:
72+
condition: service_healthy
7173
# === LABELS =============================================== #
7274
labels:
7375
- "traefik.http.routers.api.rule=PathPrefix(`/api/`)"
@@ -394,6 +396,65 @@ services:
394396
retries: 5
395397
start_period: 5s
396398

399+
seaweedfs:
400+
# === IMAGE ================================================ #
401+
# The bundled durable object store for session/agent mounts. Without it, runner mount
402+
# signing returns 503 and agent file writes are silently lost. Pinned (not :latest) so the
403+
# IAM/STS subsystem stays on a known-good build — it has regressed across releases.
404+
image: chrislusf/seaweedfs:4.37
405+
# === EXECUTION ============================================ #
406+
# ONLY FOR SEAWEEDFS (the bundled store). Real S3/R2/MinIO ship their own STS/IAM and ignore
407+
# all of this. Two configs, both generated from env (no committed files):
408+
# - s3.json: the master identity only (admin; the API holds these creds, never the runner).
409+
# - iam.json: the ADVANCED IAM config — the only path SeaweedFS authorizes STS credentials.
410+
# An OIDC provider points at the API's self-served JWKS; the API mints a short-lived
411+
# RS256 web-identity token and calls AssumeRoleWithWebIdentity to assume `agenta-store`.
412+
entrypoint:
413+
- sh
414+
- -c
415+
- |
416+
cat > /etc/seaweedfs/s3.json <<EOF
417+
{"identities":[{"name":"agenta","credentials":[{"accessKey":"$${AGENTA_STORE_ACCESS_KEY}","secretKey":"$${AGENTA_STORE_SECRET_KEY}"}],"actions":["Admin","Read","Write","List","Tagging"]}]}
418+
EOF
419+
cat > /etc/seaweedfs/iam.json <<EOF
420+
{"sts":{"tokenDuration":"1h","maxSessionLength":"12h","issuer":"seaweedfs-sts","signingKey":"$${AGENTA_STORE_SIGNING_KEY}"},"providers":[{"name":"agenta","type":"oidc","enabled":true,"config":{"issuer":"$${AGENTA_STORE_JWT_ISSUER}","clientId":"agenta-store","jwksUri":"$${AGENTA_STORE_JWT_ISSUER}/.well-known/jwks.json"}}],"policies":[{"name":"store-rw","document":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::$${AGENTA_STORE_BUCKET}","arn:aws:s3:::$${AGENTA_STORE_BUCKET}/*"]}]}}],"roles":[{"roleName":"agenta-store","roleArn":"arn:aws:iam::role/agenta-store","attachedPolicies":["store-rw"],"trustPolicy":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"agenta"},"Action":["sts:AssumeRoleWithWebIdentity"]}]}}]}
421+
EOF
422+
exec weed server -dir=/data -ip=seaweedfs -volume.max=64 -s3 -s3.port=8333 -s3.config=/etc/seaweedfs/s3.json -s3.iam.config=/etc/seaweedfs/iam.json
423+
# === CONFIGURATION ======================================== #
424+
env_file:
425+
- ${ENV_FILE:-./.env.ee.gh}
426+
environment:
427+
# The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so they must
428+
# be present in this service's env (defaults here are for the bundled store, the same way
429+
# supertokens carries its own connection URI). Keys come from the env file — secrets never
430+
# get a baked-in default. Override the whole trio via the env file to use S3/R2/MinIO.
431+
AGENTA_STORE_ACCESS_KEY: ${AGENTA_STORE_ACCESS_KEY}
432+
AGENTA_STORE_SECRET_KEY: ${AGENTA_STORE_SECRET_KEY}
433+
AGENTA_STORE_BUCKET: ${AGENTA_STORE_BUCKET:-agenta-store}
434+
AGENTA_STORE_SIGNING_KEY: ${AGENTA_STORE_SIGNING_KEY}
435+
AGENTA_STORE_JWT_ISSUER: ${AGENTA_STORE_JWT_ISSUER:-http://api:8000}
436+
# === STORAGE ============================================== #
437+
volumes:
438+
- seaweed-data:/data
439+
# === NETWORK ============================================== #
440+
networks:
441+
- agenta-ee-gh-network
442+
# Loopback-only by default (never expose the store to a public IP); override AGENTA_STORE_PORT
443+
# to change. In-network services reach it directly at seaweedfs:8333, no publish needed.
444+
ports:
445+
- "${AGENTA_STORE_PORT:-127.0.0.1:8333}:8333"
446+
# === LABELS =============================================== #
447+
labels:
448+
- "traefik.enable=false"
449+
# === LIFECYCLE ============================================ #
450+
restart: always
451+
healthcheck:
452+
test: ["CMD-SHELL", "curl -sf http://localhost:9333/cluster/healthz >/dev/null || exit 1"]
453+
interval: 5s
454+
timeout: 5s
455+
retries: 30
456+
start_period: 5s
457+
397458
traefik:
398459
# === IMAGE ================================================ #
399460
image: traefik:2
@@ -504,3 +565,4 @@ volumes:
504565
postgres-data:
505566
redis-volatile-data:
506567
redis-durable-data:
568+
seaweed-data:

‎hosting/docker-compose/ee/env.ee.gh.example‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,9 @@ AGENTA_STORE_SECRET_KEY=replace-me
329329
# AGENTA_STORE_NAMESPACE=
330330
# AGENTA_STORE_SIGNING_KEY=
331331
# AGENTA_STORE_JWT_ISSUER=http://api:8000
332+
# The bundled store's web-identity path mints an ephemeral keypair per api process, so a SINGLE
333+
# api replica works with this unset. If you scale the api past one replica, set the SAME PEM on
334+
# every replica or STS token minting fails intermittently (SeaweedFS caches one JWKS).
332335
# AGENTA_STORE_JWT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----"
333336
# AGENTA_WORKER_STREAMS / AGENTA_WORKER_QUEUES: worker topology selectors —
334337
# set inline per-service in compose, not here; see docs/designs/workers-sprawl/specs.md

‎hosting/docker-compose/oss/docker-compose.gh.local.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ services:
6666
condition: service_healthy
6767
redis-durable:
6868
condition: service_healthy
69+
seaweedfs:
70+
condition: service_healthy
6971
# === LABELS =============================================== #
7072
labels:
7173
- "traefik.http.routers.api.rule=PathPrefix(`/api/`)"
@@ -392,6 +394,65 @@ services:
392394
retries: 5
393395
start_period: 5s
394396

397+
seaweedfs:
398+
# === IMAGE ================================================ #
399+
# The bundled durable object store for session/agent mounts. Without it, runner mount
400+
# signing returns 503 and agent file writes are silently lost. Pinned (not :latest) so the
401+
# IAM/STS subsystem stays on a known-good build — it has regressed across releases.
402+
image: chrislusf/seaweedfs:4.37
403+
# === EXECUTION ============================================ #
404+
# ONLY FOR SEAWEEDFS (the bundled store). Real S3/R2/MinIO ship their own STS/IAM and ignore
405+
# all of this. Two configs, both generated from env (no committed files):
406+
# - s3.json: the master identity only (admin; the API holds these creds, never the runner).
407+
# - iam.json: the ADVANCED IAM config — the only path SeaweedFS authorizes STS credentials.
408+
# An OIDC provider points at the API's self-served JWKS; the API mints a short-lived
409+
# RS256 web-identity token and calls AssumeRoleWithWebIdentity to assume `agenta-store`.
410+
entrypoint:
411+
- sh
412+
- -c
413+
- |
414+
cat > /etc/seaweedfs/s3.json <<EOF
415+
{"identities":[{"name":"agenta","credentials":[{"accessKey":"$${AGENTA_STORE_ACCESS_KEY}","secretKey":"$${AGENTA_STORE_SECRET_KEY}"}],"actions":["Admin","Read","Write","List","Tagging"]}]}
416+
EOF
417+
cat > /etc/seaweedfs/iam.json <<EOF
418+
{"sts":{"tokenDuration":"1h","maxSessionLength":"12h","issuer":"seaweedfs-sts","signingKey":"$${AGENTA_STORE_SIGNING_KEY}"},"providers":[{"name":"agenta","type":"oidc","enabled":true,"config":{"issuer":"$${AGENTA_STORE_JWT_ISSUER}","clientId":"agenta-store","jwksUri":"$${AGENTA_STORE_JWT_ISSUER}/.well-known/jwks.json"}}],"policies":[{"name":"store-rw","document":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::$${AGENTA_STORE_BUCKET}","arn:aws:s3:::$${AGENTA_STORE_BUCKET}/*"]}]}}],"roles":[{"roleName":"agenta-store","roleArn":"arn:aws:iam::role/agenta-store","attachedPolicies":["store-rw"],"trustPolicy":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"agenta"},"Action":["sts:AssumeRoleWithWebIdentity"]}]}}]}
419+
EOF
420+
exec weed server -dir=/data -ip=seaweedfs -volume.max=64 -s3 -s3.port=8333 -s3.config=/etc/seaweedfs/s3.json -s3.iam.config=/etc/seaweedfs/iam.json
421+
# === CONFIGURATION ======================================== #
422+
env_file:
423+
- ${ENV_FILE:-./.env.oss.gh}
424+
environment:
425+
# The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so they must
426+
# be present in this service's env (defaults here are for the bundled store, the same way
427+
# supertokens carries its own connection URI). Keys come from the env file — secrets never
428+
# get a baked-in default. Override the whole trio via the env file to use S3/R2/MinIO.
429+
AGENTA_STORE_ACCESS_KEY: ${AGENTA_STORE_ACCESS_KEY}
430+
AGENTA_STORE_SECRET_KEY: ${AGENTA_STORE_SECRET_KEY}
431+
AGENTA_STORE_BUCKET: ${AGENTA_STORE_BUCKET:-agenta-store}
432+
AGENTA_STORE_SIGNING_KEY: ${AGENTA_STORE_SIGNING_KEY}
433+
AGENTA_STORE_JWT_ISSUER: ${AGENTA_STORE_JWT_ISSUER:-http://api:8000}
434+
# === STORAGE ============================================== #
435+
volumes:
436+
- seaweed-data:/data
437+
# === NETWORK ============================================== #
438+
networks:
439+
- agenta-oss-gh-network
440+
# Loopback-only by default (never expose the store to a public IP); override AGENTA_STORE_PORT
441+
# to change. In-network services reach it directly at seaweedfs:8333, no publish needed.
442+
ports:
443+
- "${AGENTA_STORE_PORT:-127.0.0.1:8333}:8333"
444+
# === LABELS =============================================== #
445+
labels:
446+
- "traefik.enable=false"
447+
# === LIFECYCLE ============================================ #
448+
restart: always
449+
healthcheck:
450+
test: ["CMD-SHELL", "curl -sf http://localhost:9333/cluster/healthz >/dev/null || exit 1"]
451+
interval: 5s
452+
timeout: 5s
453+
retries: 30
454+
start_period: 5s
455+
395456
traefik:
396457
# === ACTIVATION =========================================== #
397458
profiles:
@@ -507,3 +568,4 @@ volumes:
507568
postgres-data:
508569
redis-volatile-data:
509570
redis-durable-data:
571+
seaweed-data:

‎hosting/docker-compose/oss/docker-compose.gh.ssl.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ services:
7171
condition: service_healthy
7272
redis-durable:
7373
condition: service_healthy
74+
seaweedfs:
75+
condition: service_healthy
7476
# === LABELS =============================================== #
7577
labels:
7678
- "traefik.http.routers.api.rule=Host(`${TRAEFIK_DOMAIN}`) && PathPrefix(`/api/`)"
@@ -416,6 +418,65 @@ services:
416418
retries: 5
417419
start_period: 5s
418420

421+
seaweedfs:
422+
# === IMAGE ================================================ #
423+
# The bundled durable object store for session/agent mounts. Without it, runner mount
424+
# signing returns 503 and agent file writes are silently lost. Pinned (not :latest) so the
425+
# IAM/STS subsystem stays on a known-good build — it has regressed across releases.
426+
image: chrislusf/seaweedfs:4.37
427+
# === EXECUTION ============================================ #
428+
# ONLY FOR SEAWEEDFS (the bundled store). Real S3/R2/MinIO ship their own STS/IAM and ignore
429+
# all of this. Two configs, both generated from env (no committed files):
430+
# - s3.json: the master identity only (admin; the API holds these creds, never the runner).
431+
# - iam.json: the ADVANCED IAM config — the only path SeaweedFS authorizes STS credentials.
432+
# An OIDC provider points at the API's self-served JWKS; the API mints a short-lived
433+
# RS256 web-identity token and calls AssumeRoleWithWebIdentity to assume `agenta-store`.
434+
entrypoint:
435+
- sh
436+
- -c
437+
- |
438+
cat > /etc/seaweedfs/s3.json <<EOF
439+
{"identities":[{"name":"agenta","credentials":[{"accessKey":"$${AGENTA_STORE_ACCESS_KEY}","secretKey":"$${AGENTA_STORE_SECRET_KEY}"}],"actions":["Admin","Read","Write","List","Tagging"]}]}
440+
EOF
441+
cat > /etc/seaweedfs/iam.json <<EOF
442+
{"sts":{"tokenDuration":"1h","maxSessionLength":"12h","issuer":"seaweedfs-sts","signingKey":"$${AGENTA_STORE_SIGNING_KEY}"},"providers":[{"name":"agenta","type":"oidc","enabled":true,"config":{"issuer":"$${AGENTA_STORE_JWT_ISSUER}","clientId":"agenta-store","jwksUri":"$${AGENTA_STORE_JWT_ISSUER}/.well-known/jwks.json"}}],"policies":[{"name":"store-rw","document":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::$${AGENTA_STORE_BUCKET}","arn:aws:s3:::$${AGENTA_STORE_BUCKET}/*"]}]}}],"roles":[{"roleName":"agenta-store","roleArn":"arn:aws:iam::role/agenta-store","attachedPolicies":["store-rw"],"trustPolicy":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"agenta"},"Action":["sts:AssumeRoleWithWebIdentity"]}]}}]}
443+
EOF
444+
exec weed server -dir=/data -ip=seaweedfs -volume.max=64 -s3 -s3.port=8333 -s3.config=/etc/seaweedfs/s3.json -s3.iam.config=/etc/seaweedfs/iam.json
445+
# === CONFIGURATION ======================================== #
446+
env_file:
447+
- ${ENV_FILE:-./.env.oss.gh}
448+
environment:
449+
# The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so they must
450+
# be present in this service's env (defaults here are for the bundled store, the same way
451+
# supertokens carries its own connection URI). Keys come from the env file — secrets never
452+
# get a baked-in default. Override the whole trio via the env file to use S3/R2/MinIO.
453+
AGENTA_STORE_ACCESS_KEY: ${AGENTA_STORE_ACCESS_KEY}
454+
AGENTA_STORE_SECRET_KEY: ${AGENTA_STORE_SECRET_KEY}
455+
AGENTA_STORE_BUCKET: ${AGENTA_STORE_BUCKET:-agenta-store}
456+
AGENTA_STORE_SIGNING_KEY: ${AGENTA_STORE_SIGNING_KEY}
457+
AGENTA_STORE_JWT_ISSUER: ${AGENTA_STORE_JWT_ISSUER:-http://api:8000}
458+
# === STORAGE ============================================== #
459+
volumes:
460+
- seaweed-data:/data
461+
# === NETWORK ============================================== #
462+
networks:
463+
- agenta-gh-ssl-network
464+
# Loopback-only by default (never expose the store to a public IP); override AGENTA_STORE_PORT
465+
# to change. In-network services reach it directly at seaweedfs:8333, no publish needed.
466+
ports:
467+
- "${AGENTA_STORE_PORT:-127.0.0.1:8333}:8333"
468+
# === LABELS =============================================== #
469+
labels:
470+
- "traefik.enable=false"
471+
# === LIFECYCLE ============================================ #
472+
restart: always
473+
healthcheck:
474+
test: ["CMD-SHELL", "curl -sf http://localhost:9333/cluster/healthz >/dev/null || exit 1"]
475+
interval: 5s
476+
timeout: 5s
477+
retries: 30
478+
start_period: 5s
479+
419480
traefik:
420481
# === IMAGE ================================================ #
421482
image: traefik:2
@@ -496,3 +557,4 @@ volumes:
496557
postgres-data:
497558
redis-volatile-data:
498559
redis-durable-data:
560+
seaweed-data:

‎hosting/docker-compose/oss/env.oss.gh.example‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,9 @@ AGENTA_STORE_SECRET_KEY=replace-me
329329
# AGENTA_STORE_NAMESPACE=
330330
# AGENTA_STORE_SIGNING_KEY=
331331
# AGENTA_STORE_JWT_ISSUER=http://api:8000
332+
# The bundled store's web-identity path mints an ephemeral keypair per api process, so a SINGLE
333+
# api replica works with this unset. If you scale the api past one replica, set the SAME PEM on
334+
# every replica or STS token minting fails intermittently (SeaweedFS caches one JWKS).
332335
# AGENTA_STORE_JWT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----"
333336
# AGENTA_WORKER_STREAMS / AGENTA_WORKER_QUEUES: worker topology selectors —
334337
# set inline per-service in compose, not here; see docs/designs/workers-sprawl/specs.md

0 commit comments

Comments
 (0)