Skip to content

Commit 7df3256

Browse files
mmabroukmmabrouk
authored
Merge pull request #5315 from Agenta-AI/fix/seaweedfs-gh-variants
fix(hosting): bundle SeaweedFS store in gh.local and gh.ssl variants
2 parents 0e46146 + 00d1f8d commit 7df3256

5 files changed

Lines changed: 192 additions & 0 deletions

File tree

‎hosting/docker-compose/ee/docker-compose.gh.local.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ services:
6868
condition: service_healthy
6969
redis-durable:
7070
condition: service_healthy
71+
seaweedfs:
72+
condition: service_healthy
7173
# === LABELS =============================================== #
7274
labels:
7375
- "traefik.http.routers.api.rule=PathPrefix(`/api/`)"
@@ -396,6 +398,65 @@ services:
396398
retries: 5
397399
start_period: 5s
398400

401+
seaweedfs:
402+
# === IMAGE ================================================ #
403+
# The bundled durable object store for session/agent mounts. Without it, runner mount
404+
# signing returns 503 and agent file writes are silently lost. Pinned (not :latest) so the
405+
# IAM/STS subsystem stays on a known-good build — it has regressed across releases.
406+
image: chrislusf/seaweedfs:4.37
407+
# === EXECUTION ============================================ #
408+
# ONLY FOR SEAWEEDFS (the bundled store). Real S3/R2/MinIO ship their own STS/IAM and ignore
409+
# all of this. Two configs, both generated from env (no committed files):
410+
# - s3.json: the master identity only (admin; the API holds these creds, never the runner).
411+
# - iam.json: the ADVANCED IAM config — the only path SeaweedFS authorizes STS credentials.
412+
# An OIDC provider points at the API's self-served JWKS; the API mints a short-lived
413+
# RS256 web-identity token and calls AssumeRoleWithWebIdentity to assume `agenta-store`.
414+
entrypoint:
415+
- sh
416+
- -c
417+
- |
418+
cat > /etc/seaweedfs/s3.json <<EOF
419+
{"identities":[{"name":"agenta","credentials":[{"accessKey":"$${AGENTA_STORE_ACCESS_KEY}","secretKey":"$${AGENTA_STORE_SECRET_KEY}"}],"actions":["Admin","Read","Write","List","Tagging"]}]}
420+
EOF
421+
cat > /etc/seaweedfs/iam.json <<EOF
422+
{"sts":{"tokenDuration":"1h","maxSessionLength":"12h","issuer":"seaweedfs-sts","signingKey":"$${AGENTA_STORE_SIGNING_KEY}"},"providers":[{"name":"agenta","type":"oidc","enabled":true,"config":{"issuer":"$${AGENTA_STORE_JWT_ISSUER}","clientId":"agenta-store","jwksUri":"$${AGENTA_STORE_JWT_ISSUER}/.well-known/jwks.json"}}],"policies":[{"name":"store-rw","document":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::$${AGENTA_STORE_BUCKET}","arn:aws:s3:::$${AGENTA_STORE_BUCKET}/*"]}]}}],"roles":[{"roleName":"agenta-store","roleArn":"arn:aws:iam::role/agenta-store","attachedPolicies":["store-rw"],"trustPolicy":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"agenta"},"Action":["sts:AssumeRoleWithWebIdentity"]}]}}]}
423+
EOF
424+
exec weed server -dir=/data -ip=seaweedfs -volume.max=64 -s3 -s3.port=8333 -s3.config=/etc/seaweedfs/s3.json -s3.iam.config=/etc/seaweedfs/iam.json
425+
# === CONFIGURATION ======================================== #
426+
env_file:
427+
- ${ENV_FILE:-./.env.ee.gh}
428+
environment:
429+
# The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so they must
430+
# be present in this service's env (defaults here are for the bundled store, the same way
431+
# supertokens carries its own connection URI). Keys come from the env file — secrets never
432+
# get a baked-in default. Override the whole trio via the env file to use S3/R2/MinIO.
433+
AGENTA_STORE_ACCESS_KEY: ${AGENTA_STORE_ACCESS_KEY}
434+
AGENTA_STORE_SECRET_KEY: ${AGENTA_STORE_SECRET_KEY}
435+
AGENTA_STORE_BUCKET: ${AGENTA_STORE_BUCKET:-agenta-store}
436+
AGENTA_STORE_SIGNING_KEY: ${AGENTA_STORE_SIGNING_KEY}
437+
AGENTA_STORE_JWT_ISSUER: ${AGENTA_STORE_JWT_ISSUER:-http://api:8000}
438+
# === STORAGE ============================================== #
439+
volumes:
440+
- seaweed-data:/data
441+
# === NETWORK ============================================== #
442+
networks:
443+
- agenta-ee-gh-network
444+
# Loopback-only by default (never expose the store to a public IP); override AGENTA_STORE_PORT
445+
# to change. In-network services reach it directly at seaweedfs:8333, no publish needed.
446+
ports:
447+
- "${AGENTA_STORE_PORT:-127.0.0.1:8333}:8333"
448+
# === LABELS =============================================== #
449+
labels:
450+
- "traefik.enable=false"
451+
# === LIFECYCLE ============================================ #
452+
restart: always
453+
healthcheck:
454+
test: ["CMD-SHELL", "curl -sf http://localhost:9333/cluster/healthz >/dev/null || exit 1"]
455+
interval: 5s
456+
timeout: 5s
457+
retries: 30
458+
start_period: 5s
459+
399460
traefik:
400461
# === IMAGE ================================================ #
401462
image: traefik:2
@@ -506,3 +567,4 @@ volumes:
506567
postgres-data:
507568
redis-volatile-data:
508569
redis-durable-data:
570+
seaweed-data:

‎hosting/docker-compose/ee/env.ee.gh.example‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,9 @@ AGENTA_STORE_SIGNING_KEY=replace-me
372372
# AGENTA_STORE_REGION=us-east-1
373373
# AGENTA_STORE_NAMESPACE=
374374
# AGENTA_STORE_JWT_ISSUER=http://api:8000
375+
# The bundled store's web-identity path mints an ephemeral keypair per api process, so a SINGLE
376+
# api replica works with this unset. If you scale the api past one replica, set the SAME PEM on
377+
# every replica or STS token minting fails intermittently (SeaweedFS caches one JWKS).
375378
# AGENTA_STORE_JWT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----"
376379
# AGENTA_WORKER_STREAMS / AGENTA_WORKER_QUEUES: worker topology selectors —
377380
# set inline per-service in compose, not here; see docs/designs/workers-sprawl/specs.md

‎hosting/docker-compose/oss/docker-compose.gh.local.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ services:
6666
condition: service_healthy
6767
redis-durable:
6868
condition: service_healthy
69+
seaweedfs:
70+
condition: service_healthy
6971
# === LABELS =============================================== #
7072
labels:
7173
- "traefik.http.routers.api.rule=PathPrefix(`/api/`)"
@@ -394,6 +396,65 @@ services:
394396
retries: 5
395397
start_period: 5s
396398

399+
seaweedfs:
400+
# === IMAGE ================================================ #
401+
# The bundled durable object store for session/agent mounts. Without it, runner mount
402+
# signing returns 503 and agent file writes are silently lost. Pinned (not :latest) so the
403+
# IAM/STS subsystem stays on a known-good build — it has regressed across releases.
404+
image: chrislusf/seaweedfs:4.37
405+
# === EXECUTION ============================================ #
406+
# ONLY FOR SEAWEEDFS (the bundled store). Real S3/R2/MinIO ship their own STS/IAM and ignore
407+
# all of this. Two configs, both generated from env (no committed files):
408+
# - s3.json: the master identity only (admin; the API holds these creds, never the runner).
409+
# - iam.json: the ADVANCED IAM config — the only path SeaweedFS authorizes STS credentials.
410+
# An OIDC provider points at the API's self-served JWKS; the API mints a short-lived
411+
# RS256 web-identity token and calls AssumeRoleWithWebIdentity to assume `agenta-store`.
412+
entrypoint:
413+
- sh
414+
- -c
415+
- |
416+
cat > /etc/seaweedfs/s3.json <<EOF
417+
{"identities":[{"name":"agenta","credentials":[{"accessKey":"$${AGENTA_STORE_ACCESS_KEY}","secretKey":"$${AGENTA_STORE_SECRET_KEY}"}],"actions":["Admin","Read","Write","List","Tagging"]}]}
418+
EOF
419+
cat > /etc/seaweedfs/iam.json <<EOF
420+
{"sts":{"tokenDuration":"1h","maxSessionLength":"12h","issuer":"seaweedfs-sts","signingKey":"$${AGENTA_STORE_SIGNING_KEY}"},"providers":[{"name":"agenta","type":"oidc","enabled":true,"config":{"issuer":"$${AGENTA_STORE_JWT_ISSUER}","clientId":"agenta-store","jwksUri":"$${AGENTA_STORE_JWT_ISSUER}/.well-known/jwks.json"}}],"policies":[{"name":"store-rw","document":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::$${AGENTA_STORE_BUCKET}","arn:aws:s3:::$${AGENTA_STORE_BUCKET}/*"]}]}}],"roles":[{"roleName":"agenta-store","roleArn":"arn:aws:iam::role/agenta-store","attachedPolicies":["store-rw"],"trustPolicy":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"agenta"},"Action":["sts:AssumeRoleWithWebIdentity"]}]}}]}
421+
EOF
422+
exec weed server -dir=/data -ip=seaweedfs -volume.max=64 -s3 -s3.port=8333 -s3.config=/etc/seaweedfs/s3.json -s3.iam.config=/etc/seaweedfs/iam.json
423+
# === CONFIGURATION ======================================== #
424+
env_file:
425+
- ${ENV_FILE:-./.env.oss.gh}
426+
environment:
427+
# The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so they must
428+
# be present in this service's env (defaults here are for the bundled store, the same way
429+
# supertokens carries its own connection URI). Keys come from the env file — secrets never
430+
# get a baked-in default. Override the whole trio via the env file to use S3/R2/MinIO.
431+
AGENTA_STORE_ACCESS_KEY: ${AGENTA_STORE_ACCESS_KEY}
432+
AGENTA_STORE_SECRET_KEY: ${AGENTA_STORE_SECRET_KEY}
433+
AGENTA_STORE_BUCKET: ${AGENTA_STORE_BUCKET:-agenta-store}
434+
AGENTA_STORE_SIGNING_KEY: ${AGENTA_STORE_SIGNING_KEY}
435+
AGENTA_STORE_JWT_ISSUER: ${AGENTA_STORE_JWT_ISSUER:-http://api:8000}
436+
# === STORAGE ============================================== #
437+
volumes:
438+
- seaweed-data:/data
439+
# === NETWORK ============================================== #
440+
networks:
441+
- agenta-oss-gh-network
442+
# Loopback-only by default (never expose the store to a public IP); override AGENTA_STORE_PORT
443+
# to change. In-network services reach it directly at seaweedfs:8333, no publish needed.
444+
ports:
445+
- "${AGENTA_STORE_PORT:-127.0.0.1:8333}:8333"
446+
# === LABELS =============================================== #
447+
labels:
448+
- "traefik.enable=false"
449+
# === LIFECYCLE ============================================ #
450+
restart: always
451+
healthcheck:
452+
test: ["CMD-SHELL", "curl -sf http://localhost:9333/cluster/healthz >/dev/null || exit 1"]
453+
interval: 5s
454+
timeout: 5s
455+
retries: 30
456+
start_period: 5s
457+
397458
traefik:
398459
# === ACTIVATION =========================================== #
399460
profiles:
@@ -509,3 +570,4 @@ volumes:
509570
postgres-data:
510571
redis-volatile-data:
511572
redis-durable-data:
573+
seaweed-data:

‎hosting/docker-compose/oss/docker-compose.gh.ssl.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ services:
7171
condition: service_healthy
7272
redis-durable:
7373
condition: service_healthy
74+
seaweedfs:
75+
condition: service_healthy
7476
# === LABELS =============================================== #
7577
labels:
7678
- "traefik.http.routers.api.rule=Host(`${TRAEFIK_DOMAIN}`) && PathPrefix(`/api/`)"
@@ -418,6 +420,65 @@ services:
418420
retries: 5
419421
start_period: 5s
420422

423+
seaweedfs:
424+
# === IMAGE ================================================ #
425+
# The bundled durable object store for session/agent mounts. Without it, runner mount
426+
# signing returns 503 and agent file writes are silently lost. Pinned (not :latest) so the
427+
# IAM/STS subsystem stays on a known-good build — it has regressed across releases.
428+
image: chrislusf/seaweedfs:4.37
429+
# === EXECUTION ============================================ #
430+
# ONLY FOR SEAWEEDFS (the bundled store). Real S3/R2/MinIO ship their own STS/IAM and ignore
431+
# all of this. Two configs, both generated from env (no committed files):
432+
# - s3.json: the master identity only (admin; the API holds these creds, never the runner).
433+
# - iam.json: the ADVANCED IAM config — the only path SeaweedFS authorizes STS credentials.
434+
# An OIDC provider points at the API's self-served JWKS; the API mints a short-lived
435+
# RS256 web-identity token and calls AssumeRoleWithWebIdentity to assume `agenta-store`.
436+
entrypoint:
437+
- sh
438+
- -c
439+
- |
440+
cat > /etc/seaweedfs/s3.json <<EOF
441+
{"identities":[{"name":"agenta","credentials":[{"accessKey":"$${AGENTA_STORE_ACCESS_KEY}","secretKey":"$${AGENTA_STORE_SECRET_KEY}"}],"actions":["Admin","Read","Write","List","Tagging"]}]}
442+
EOF
443+
cat > /etc/seaweedfs/iam.json <<EOF
444+
{"sts":{"tokenDuration":"1h","maxSessionLength":"12h","issuer":"seaweedfs-sts","signingKey":"$${AGENTA_STORE_SIGNING_KEY}"},"providers":[{"name":"agenta","type":"oidc","enabled":true,"config":{"issuer":"$${AGENTA_STORE_JWT_ISSUER}","clientId":"agenta-store","jwksUri":"$${AGENTA_STORE_JWT_ISSUER}/.well-known/jwks.json"}}],"policies":[{"name":"store-rw","document":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::$${AGENTA_STORE_BUCKET}","arn:aws:s3:::$${AGENTA_STORE_BUCKET}/*"]}]}}],"roles":[{"roleName":"agenta-store","roleArn":"arn:aws:iam::role/agenta-store","attachedPolicies":["store-rw"],"trustPolicy":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":"agenta"},"Action":["sts:AssumeRoleWithWebIdentity"]}]}}]}
445+
EOF
446+
exec weed server -dir=/data -ip=seaweedfs -volume.max=64 -s3 -s3.port=8333 -s3.config=/etc/seaweedfs/s3.json -s3.iam.config=/etc/seaweedfs/iam.json
447+
# === CONFIGURATION ======================================== #
448+
env_file:
449+
- ${ENV_FILE:-./.env.oss.gh}
450+
environment:
451+
# The entrypoint bakes these into s3.json/iam.json via raw shell expansion, so they must
452+
# be present in this service's env (defaults here are for the bundled store, the same way
453+
# supertokens carries its own connection URI). Keys come from the env file — secrets never
454+
# get a baked-in default. Override the whole trio via the env file to use S3/R2/MinIO.
455+
AGENTA_STORE_ACCESS_KEY: ${AGENTA_STORE_ACCESS_KEY}
456+
AGENTA_STORE_SECRET_KEY: ${AGENTA_STORE_SECRET_KEY}
457+
AGENTA_STORE_BUCKET: ${AGENTA_STORE_BUCKET:-agenta-store}
458+
AGENTA_STORE_SIGNING_KEY: ${AGENTA_STORE_SIGNING_KEY}
459+
AGENTA_STORE_JWT_ISSUER: ${AGENTA_STORE_JWT_ISSUER:-http://api:8000}
460+
# === STORAGE ============================================== #
461+
volumes:
462+
- seaweed-data:/data
463+
# === NETWORK ============================================== #
464+
networks:
465+
- agenta-gh-ssl-network
466+
# Loopback-only by default (never expose the store to a public IP); override AGENTA_STORE_PORT
467+
# to change. In-network services reach it directly at seaweedfs:8333, no publish needed.
468+
ports:
469+
- "${AGENTA_STORE_PORT:-127.0.0.1:8333}:8333"
470+
# === LABELS =============================================== #
471+
labels:
472+
- "traefik.enable=false"
473+
# === LIFECYCLE ============================================ #
474+
restart: always
475+
healthcheck:
476+
test: ["CMD-SHELL", "curl -sf http://localhost:9333/cluster/healthz >/dev/null || exit 1"]
477+
interval: 5s
478+
timeout: 5s
479+
retries: 30
480+
start_period: 5s
481+
421482
traefik:
422483
# === IMAGE ================================================ #
423484
image: traefik:2
@@ -498,3 +559,4 @@ volumes:
498559
postgres-data:
499560
redis-volatile-data:
500561
redis-durable-data:
562+
seaweed-data:

‎hosting/docker-compose/oss/env.oss.gh.example‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,9 @@ AGENTA_STORE_SIGNING_KEY=replace-me
372372
# AGENTA_STORE_REGION=us-east-1
373373
# AGENTA_STORE_NAMESPACE=
374374
# AGENTA_STORE_JWT_ISSUER=http://api:8000
375+
# The bundled store's web-identity path mints an ephemeral keypair per api process, so a SINGLE
376+
# api replica works with this unset. If you scale the api past one replica, set the SAME PEM on
377+
# every replica or STS token minting fails intermittently (SeaweedFS caches one JWKS).
375378
# AGENTA_STORE_JWT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----"
376379
# AGENTA_WORKER_STREAMS / AGENTA_WORKER_QUEUES: worker topology selectors —
377380
# set inline per-service in compose, not here; see docs/designs/workers-sprawl/specs.md

0 commit comments

Comments
 (0)