From be49aedda9315b8edceddd4d9f2aee370b757c21 Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Sat, 4 Apr 2026 03:48:44 +0200 Subject: [PATCH] chore: update compose stacks and shared networking - Upgrade PostgreSQL 16 -> 17 - Upgrade CockroachDB v22.2.2 -> v22.2.15 - Upgrade substrate-gateway 2.5.0 -> 2.7.0 - Rename indexer CockroachDB service from 'db' to 'cockroachdb' to avoid DNS collision when both stacks share a Docker network - Add shared external network 'tfgrid_bknd' to both compose files so the processor can reach the indexer gateway by service name - Update INDEXER_ENDPOINT_URL default to http://gateway:8000/graphql (Docker service name) instead of localhost:8888 (host port mapping) - Bind database ports to localhost only (PostgreSQL 5432, CockroachDB 26257, CockroachDB admin 8080) to prevent unauthorized access - Remove obsolete 'version' field from both compose files - Update production docs with shared network setup and env var changes Co-Authored-By: Claude Opus 4.6 (1M context) --- .env | 7 ++++--- docker-compose.yml | 9 ++++++--- docs/development.md | 18 ++++++++++++++++- docs/production.md | 36 ++++++++++++++++++++++++++++++++-- indexer/docker-compose.yml | 40 ++++++++++++++++++++++---------------- indexer/readme.md | 28 +++++++++++++++++++++++--- indexer/setup_db.sh | 2 +- 7 files changed, 110 insertions(+), 30 deletions(-) diff --git a/.env b/.env index 73d9a56..3cbd590 100644 --- a/.env +++ b/.env @@ -1,8 +1,9 @@ DB_NAME=tfgrid-graphql DB_USER=postgres DB_PASS=postgres -#DB_HOST=172.17.0.1 DB_PORT=5432 TYPEORM_LOGGING=error -INDEXER_ENDPOINT_URL=http://localhost:8888/graphql -WS_URL=ws://localhost:9944 +# For Docker deployment (shared tfgrid_bknd network): use gateway service name +# For local development outside Docker: use http://localhost:8888/graphql +INDEXER_ENDPOINT_URL=http://gateway:8000/graphql +WS_URL=wss://tfchain.dev.grid.tf/ws diff --git a/docker-compose.yml b/docker-compose.yml index dea3d0b..5f37746 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,14 @@ -version: "3.4" +networks: + default: + name: tfgrid_bknd + external: true services: db: - image: postgres:16 + image: postgres:17 restart: always ports: - - "${DB_PORT}:5432" + - "127.0.0.1:${DB_PORT}:5432" volumes: - /var/lib/postgresql/data environment: diff --git a/docs/development.md b/docs/development.md index a27c310..8fd0f2d 100644 --- a/docs/development.md +++ b/docs/development.md @@ -13,6 +13,14 @@ yarn build See https://github.com/threefoldtech/tfchain +### Create the shared Docker network + +Both compose stacks use a shared external network. Create it once: + +```bash +docker network create tfgrid_bknd +``` + ### Run Indexer Check `indexer/.env` and adjust the websocket endpoint to your local TFChain address. @@ -34,7 +42,15 @@ You should see TFChain blocks being processed: ### Run Processor (local, outside Docker) -Check `.env` and adjust the websocket endpoint and indexer URL to your local setup. +Check `.env` and adjust the settings. When running the processor locally (not in Docker) while the indexer runs in Docker, change `INDEXER_ENDPOINT_URL` to use the published port: + +```bash +# .env — for local development (processor outside Docker): +INDEXER_ENDPOINT_URL=http://localhost:8888/graphql + +# .env — for Docker deployment (processor inside Docker, shared network): +# INDEXER_ENDPOINT_URL=http://gateway:8000/graphql +``` Start the local PostgreSQL container and run the processor: diff --git a/docs/production.md b/docs/production.md index a8e4eb7..26ce887 100644 --- a/docs/production.md +++ b/docs/production.md @@ -15,8 +15,40 @@ The production stack has two independent layers: ## Run the Setup +Both compose stacks share a Docker network (`tfgrid_bknd`) so the processor can reach the indexer gateway by service name. Create it before starting either stack: + +```bash +docker network create tfgrid_bknd +``` + ### 1. Indexer +#### Using a snapshot (recommended) + +Download a pre-built CockroachDB snapshot to avoid syncing from scratch (which takes days): + +```bash +# Check https://bknd.snapshot.grid.tf/ for latest snapshots per network +wget -O snapshot.tar.gz + +# Extract — do NOT use --strip-components +# (SST files are at root level; --strip-components silently skips them all) +mkdir -p /path/to/cockroach-data +tar -xzf snapshot.tar.gz -C /path/to/cockroach-data/ + +# Verify: should see thousands of .sst files +find /path/to/cockroach-data/ -name "*.sst" | wc -l +``` + +Bind-mount the data directory in `indexer/docker-compose.yml`: +```yaml +cockroachdb: + volumes: + - /path/to/cockroach-data:/cockroach/cockroach-data +``` + +#### Configuration + Configure `indexer/.env`: | Variable | Description | Default | @@ -41,8 +73,8 @@ Configure `.env` in the project root: | `DB_USER` | PostgreSQL user | `postgres` | | `DB_PASS` | PostgreSQL password | `postgres` | | `DB_PORT` | PostgreSQL port | `5432` | -| `INDEXER_ENDPOINT_URL` | Indexer GraphQL gateway URL | `http://localhost:8888/graphql` | -| `WS_URL` | TFChain node WebSocket URL (used for RPC calls) | `ws://localhost:9944` | +| `INDEXER_ENDPOINT_URL` | Indexer GraphQL gateway URL. Use `http://gateway:8000/graphql` for Docker (shared network), `http://localhost:8888/graphql` for local dev | `http://gateway:8000/graphql` | +| `WS_URL` | TFChain node WebSocket URL (used for RPC calls) | `wss://tfchain.dev.grid.tf/ws` | | `TYPEORM_LOGGING` | TypeORM log level | `error` | ```bash diff --git a/indexer/docker-compose.yml b/indexer/docker-compose.yml index 37bd073..a85bc1a 100644 --- a/indexer/docker-compose.yml +++ b/indexer/docker-compose.yml @@ -1,30 +1,32 @@ -version: "3.8" +networks: + default: + name: tfgrid_bknd + external: true services: - db: - image: cockroachdb/cockroach:v22.2.2 + cockroachdb: + image: cockroachdb/cockroach:v22.2.15 restart: always ports: - - "26257:26257" - - "8080:8080" + - "127.0.0.1:26257:26257" + - "127.0.0.1:8080:8080" command: start-single-node --insecure volumes: - /cockroach/cockroach-data - db-init: + cockroachdb-init: depends_on: - - db - image: cockroachdb/cockroach:v22.2.2 - + - cockroachdb + image: cockroachdb/cockroach:v22.2.15 volumes: - ./setup_db.sh:/setup_db.sh entrypoint: "/bin/bash" command: /setup_db.sh restart: on-failure - + ingest: depends_on: - db-init: + cockroachdb-init: condition: service_completed_successfully restart: on-failure image: subsquid/substrate-ingest:1 @@ -37,7 +39,7 @@ services: "-c", "20", "--out", - "postgres://root@db:26257/defaultdb", + "postgres://root@cockroachdb:26257/defaultdb", "--types-bundle", "/configs/typesBundle.json", "--start-block", @@ -46,24 +48,28 @@ services: gateway: depends_on: - db-init: + cockroachdb-init: condition: service_completed_successfully - image: subsquid/substrate-gateway:2.5.0 + image: subsquid/substrate-gateway:2.7.0 environment: DATABASE_MAX_CONNECTIONS: 5 RUST_LOG: "actix_web=info,actix_server=info" - command: [ "--database-url", "postgres://root@db:26257/defaultdb" ] + command: [ "--database-url", "postgres://root@cockroachdb:26257/defaultdb" ] + # Published for local dev (localhost:8888) and browser access. + # In production, the processor reaches the gateway via service name + # (http://gateway:8000/graphql) on the shared network. Comment out + # these ports if you don't need host access. ports: - "8888:8000" explorer: depends_on: - db-init: + cockroachdb-init: condition: service_completed_successfully image: subsquid/substrate-explorer:firesquid environment: DB_TYPE: cockroach - DB_HOST: db + DB_HOST: cockroachdb DB_PORT: "26257" DB_NAME: "defaultdb" DB_USER: "root" diff --git a/indexer/readme.md b/indexer/readme.md index 69a7e15..b6c0142 100644 --- a/indexer/readme.md +++ b/indexer/readme.md @@ -1,5 +1,13 @@ # Setting Up an Indexer +## Prerequisites + +Both the indexer and processor compose stacks share a Docker network. Create it before starting: + +```bash +docker network create tfgrid_bknd +``` + ## Configuration The `.env` file contains the indexer options: @@ -18,13 +26,13 @@ START_HEIGHT=0 Start the indexer stack: -``` +```bash docker compose up -d ``` Stop: -``` +```bash docker compose down ``` @@ -32,9 +40,23 @@ docker compose down | Container | Image | Role | |-----------|-------|------| -| db | `cockroachdb/cockroach` | Database for storing raw indexed block data | +| cockroachdb | `cockroachdb/cockroach` | Database for storing raw indexed block data | | ingest | `subsquid/substrate-ingest` | Connects to the TFChain node and ingests blocks into the database | | gateway | `subsquid/substrate-gateway` | GraphQL gateway over ingested data — the processor queries this | | explorer | `subsquid/substrate-explorer` | Web UI to browse raw ingested data and check sync status | +### CockroachDB Memory Tuning + +By default, CockroachDB uses up to 25% of system RAM for cache and SQL memory. On VMs with limited RAM (4 GB or less), this can cause OOM. Add `--cache` and `--max-sql-memory` flags to the compose command to control usage: + +```yaml +# Fixed sizes for small VMs: +command: start-single-node --insecure --cache=256MiB --max-sql-memory=256MiB + +# Or fractions of total RAM for production (8+ GB): +command: start-single-node --insecure --cache=.25 --max-sql-memory=.25 +``` + +If you see `memory budget exceeded` errors from the gateway or explorer, increase `--max-sql-memory`. + **Note on CockroachDB:** The `--insecure` flag is used for non-production/testing only. For production, use a secure cluster. See [CockroachDB docs](https://www.cockroachlabs.com/docs/stable/deploy-cockroachdb-on-premises-insecure). diff --git a/indexer/setup_db.sh b/indexer/setup_db.sh index 2435423..6e3a89b 100644 --- a/indexer/setup_db.sh +++ b/indexer/setup_db.sh @@ -2,7 +2,7 @@ sleep 2 echo "Set sql.conn.max_read_buffer_message_size to 32 MiB" -HOSTPARAMS="--host db --insecure" +HOSTPARAMS="--host cockroachdb --insecure" SQL="/cockroach/cockroach.sh sql $HOSTPARAMS" # https://github.com/threefoldtech/tfchain_graphql/issues/130#issuecomment-1689987550