update docker compose file, etcd is not used anymore for met storage

Author: danrusei

docker/README.md

@@ -20,7 +20,7 @@ docker compose up -d
 
 ```bash
 docker compose ps
-# Both etcd and broker should show "Up" / "healthy"
+# The broker should show "Up" / "healthy"
 ```
 
 ## Run Integration Tests
@@ -44,7 +44,7 @@ docker compose down -v   # -v removes volumes for a fresh start
 
 ## Architecture
 
-- **ETCD** — metadata store on port 2379
+- **Embedded Raft metadata store** — no external dependency required
 - **Single broker** — gRPC on port 6650, admin on 50051
 - No TLS, no auth, filesystem backend
 - Broker advertises as `127.0.0.1:6650` so topic lookups resolve from the host

docker/danube_broker.yml

@@ -10,13 +10,16 @@ broker:
     client: 6650
     # Admin API
     admin: 50051
+    # Raft inter-node gRPC transport
+    raft: 7650
     # Prometheus metrics exporter
     prometheus: 9040
 
-# Metadata Store Configuration (etcd)
+# Metadata Store Configuration (Raft — embedded, no external dependency)
+# node_id is auto-generated on first boot and persisted in {data_dir}/node_id
 meta_store:
-  host: "etcd"
-  port: 2379
+  data_dir: "./danube-data/raft"
+  # Empty or omitted = single-node cluster (auto-init, zero config).
 
 # Namespaces to be created on boot
 bootstrap_namespaces:
@@ -52,7 +55,7 @@ load_manager:
   assignment_strategy: "fair"
 
   # LoadReport Publishing Interval: How often brokers publish load reports (seconds)
-  # - Lower values (5-10s): Faster response, more etcd traffic, better for testing
+  # - Lower values (5-10s): Faster response, more metadata traffic, better for testing
   # - Higher values (30-60s): Less overhead, suitable for stable production clusters
   # Default: 30 seconds
   load_report_interval_seconds: 30
@@ -156,7 +159,7 @@ wal_cloud:
     # Testing: 30s. Production: 300s (5 min). High-durability: 60s (1 min)
     interval_seconds: 30
 
-    # ETCD metadata root prefix (change only for multiple independent clusters)
+    # Metadata root prefix (change only for multiple independent clusters)
     root_prefix: "/danube-data"
 
     # Max cloud object size (MB) - optimal for S3/GCS multipart uploads
@@ -175,14 +178,6 @@ wal_cloud:
 
     # Backend-specific options - see commented examples below
 
-  # Metadata Store: ETCD for cloud object descriptors and indexes
-  metadata:
-    # ETCD endpoint (should match broker's meta_store for consistency)
-    etcd_endpoint: "127.0.0.1:2379"
-
-    # In-memory metadata (testing only, no durability)
-    in_memory: false
-
 # Broker policies, that can be overwritten by namespace / topic policies
 policies:
   # Limits the maximum number of producers that can simultaneously publish messages to a specific topic.

docker/docker-compose.yml

@@ -1,49 +1,13 @@
 # Danube single-broker setup for integration testing
-# Usage: docker-compose up -d && go test ./integration_tests/ -v -count=1
+# Usage: docker compose up -d && mvn -pl danube-client verify -P integration-tests
 
 services:
-  etcd:
-    container_name: "danube-test-etcd"
-    image: quay.io/coreos/etcd:v3.5.9
-    environment:
-      - ETCDCTL_API=3
-      - ETCD_NAME=danube-etcd
-      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
-      - ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379
-      - ETCD_INITIAL_CLUSTER=danube-etcd=http://etcd:2380
-      - ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380
-      - ETCD_INITIAL_ADVERTISE_PEER_URLS=http://etcd:2380
-      - ETCD_INITIAL_CLUSTER_STATE=new
-      - ETCD_INITIAL_CLUSTER_TOKEN=danube-test
-    ports:
-      - "2379:2379"
-    networks:
-      - danube_test_net
-    volumes:
-      - etcd_data:/etcd-data
-    healthcheck:
-      test:
-        [
-          "CMD",
-          "etcdctl",
-          "--endpoints=http://127.0.0.1:2379",
-          "endpoint",
-          "health",
-        ]
-      interval: 5s
-      timeout: 5s
-      retries: 10
-      start_period: 10s
-
   broker:
     container_name: "danube-test-broker"
     image: ghcr.io/danube-messaging/danube-broker:latest
     restart: on-failure
-    depends_on:
-      etcd:
-        condition: service_healthy
     environment:
-      - RUST_LOG=danube_broker=info,danube_core=info
+      - RUST_LOG=danube_broker=info,danube_core=info,openraft=warn
     volumes:
       - ./danube_broker.yml:/etc/danube_broker.yml:ro
       - broker_data:/danube-data
@@ -61,6 +25,8 @@ services:
         "0.0.0.0:6650",
         "--admin-addr",
         "0.0.0.0:50051",
+        "--raft-addr",
+        "0.0.0.0:7650",
         "--advertised-addr",
         "127.0.0.1:6650",
         "--prom-exporter",
@@ -79,7 +45,5 @@ networks:
     driver: bridge
 
 volumes:
-  etcd_data:
-    name: danube_test_etcd_data
   broker_data:
     name: danube_test_broker_data