diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..9d902395 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,44 @@ +# Build artifacts +target/ +**/target/ + +# Local environments +.venv/ +venv/ +__pycache__/ +*.pyc +.pytest_cache/ + +# Editor / OS +.vscode/ +.idea/ +*.swp +.DS_Store + +# Generated / runtime +*.log +discussions/ +docs/pdfs/ +docs/rendered/ +pdfs/ + +# Local config (must NOT be baked into the image; mounted at /etc/extenddb) +extenddb.toml +external-suites.toml + +# Tests are not needed for the runtime image. The Dockerfile build stage +# uses cargo workspace metadata only; integration tests run separately. +tests/ + +# Container artifacts (avoid recursive context inclusion) +# We DO want docker/entrypoint.sh in the context — only ignore the Dockerfile +# itself and the samples/docker dir, which holds compose and demo docs. +Dockerfile +.dockerignore +samples/docker/ + +# VCS +# .git/ is intentionally NOT ignored: crates/bin/build.rs runs `git rev-parse` +# during the build stage to bake the commit hash into the binary. Excluding +# .git would yield `commit unknown` in `extenddb version`. +.github/ diff --git a/.gitignore b/.gitignore index a57cf7aa..3d5123a0 100755 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ pdfs/ docs/rendered/ discussions/ .DS_Store + +# Generated by samples/docker/bootstrap-iam.sh +samples/docker/extenddb-creds.env +samples/docker/extenddb-cert.pem diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..398fda76 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,115 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Multi-stage build for ExtendDB. +# +# Stage 1 (builder): cargo build --release --bin extenddb +# Stage 2 (runtime): debian:bookworm-slim with the static-ish binary +# plus tini, ca-certificates, and curl for healthchecks +# +# This image runs `extenddb serve` only. Operator runs `extenddb init` +# explicitly as a separate one-shot before first start. See +# samples/docker/README.md for the bootstrap walkthrough. +# +# Build: +# docker build -t extenddb:dev . +# +# Run (after init): +# docker run --rm -p 8000:8000 \ +# -v extenddb-config:/etc/extenddb \ +# -v extenddb-state:/var/lib/extenddb \ +# extenddb:dev + +# ---- Stage 1: builder ---- +FROM rust:1.88-bookworm AS builder + +WORKDIR /src + +# Install git so build.rs can read the commit hash. +# pkg-config and libssl-dev are NOT required: ExtendDB uses rustls. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Copy the entire workspace. .dockerignore strips target/, tests/, docs/, +# samples/, devtools/, .venv/, etc. +COPY . . + +# Cache cargo registry across builds via BuildKit. +# Build only the binary; library crates are pulled in transitively. +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/src/target \ + cargo build --release --bin extenddb \ + && cp /src/target/release/extenddb /usr/local/bin/extenddb \ + && /usr/local/bin/extenddb --version + +# ---- Stage 2: runtime ---- +FROM debian:bookworm-slim AS runtime + +ARG EXTENDDB_UID=1000 +ARG EXTENDDB_GID=1000 + +# Runtime dependencies: +# ca-certificates: for outbound TLS (e.g. to RDS). Server-side TLS uses rustls and needs no system roots. +# tini: PID 1 reaper, signal forwarding. +# curl: HEALTHCHECK uses it to hit /health. ~250 KB. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + tini \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Non-root user. HOME=/var/lib/extenddb so init resolves ~/.extenddb +# to a writeable, container-friendly state directory. +RUN groupadd --system --gid ${EXTENDDB_GID} extenddb \ + && useradd --system --uid ${EXTENDDB_UID} --gid extenddb \ + --home-dir /var/lib/extenddb --shell /usr/sbin/nologin extenddb \ + && mkdir -p /etc/extenddb /var/lib/extenddb \ + && chown -R extenddb:extenddb /etc/extenddb /var/lib/extenddb \ + && chmod 0750 /etc/extenddb /var/lib/extenddb + +COPY --from=builder /usr/local/bin/extenddb /usr/local/bin/extenddb +COPY docker/entrypoint.sh /usr/local/bin/extenddb-entrypoint +RUN chmod 0755 /usr/local/bin/extenddb /usr/local/bin/extenddb-entrypoint + +USER extenddb +WORKDIR /var/lib/extenddb +ENV HOME=/var/lib/extenddb + +# Default DynamoDB API port. Operator can override with EXTENDDB__SERVER__PORT +# at serve time and remap the published port. +EXPOSE 8000 + +# Container runtimes default to SIGTERM, but stating it explicitly is a +# defence against future Docker default changes. extenddb honors SIGTERM +# (the entrypoint forwards it to the daemon). +STOPSIGNAL SIGTERM + +# State volumes: +# /etc/extenddb: owns extenddb.toml (written by `init`, read by `serve`) +# /var/lib/extenddb: owns ~/.extenddb/{tls,run} state +VOLUME ["/etc/extenddb", "/var/lib/extenddb"] + +# Healthcheck: curl with -k since the cert is self-signed by default. +# Operators with a CA-signed cert can override at run time. +HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=3 \ + CMD curl -kfsS https://127.0.0.1:8000/health || exit 1 + +# tini reaps zombies and forwards signals to the entrypoint script. +# Default CMD is "serve"; pass any other extenddb subcommand (e.g. +# docker run ... extenddb init --pg-host postgres ... +# ) and the entrypoint execs it directly. +ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/extenddb-entrypoint"] +CMD ["serve"] + +# OCI labels (org.opencontainers.image.*). The CI workflow overrides these +# via docker/metadata-action with revision/version/created at publish time; +# the values here are the static fallback for local builds. +LABEL org.opencontainers.image.title="ExtendDB" \ + org.opencontainers.image.description="DynamoDB-compatible API server backed by PostgreSQL" \ + org.opencontainers.image.url="https://github.com/ExtendDB/extenddb" \ + org.opencontainers.image.source="https://github.com/ExtendDB/extenddb" \ + org.opencontainers.image.documentation="https://github.com/ExtendDB/extenddb/tree/main/docs" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.vendor="ExtendDB contributors" diff --git a/README.md b/README.md index 5e5c76a6..3f05aa0f 100755 --- a/README.md +++ b/README.md @@ -47,6 +47,21 @@ scripts/install-linux.sh # Linux scripts/install-macos.sh # macOS ``` +### Run with Docker + +A Docker Compose stack brings up PostgreSQL and ExtendDB end-to-end: + +```bash +cd samples/docker +docker compose -f compose.yaml -f compose.dev.yaml up --build -d +./bootstrap-iam.sh # creates an IAM user + access key +source ./extenddb-creds.env +aws dynamodb list-tables --endpoint-url "$EXTENDDB_ENDPOINT" +``` + +See [`samples/docker/README.md`](samples/docker/README.md) for the full +walkthrough. + ## Prerequisites - Rust 1.85+ (`rustup update`) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000..d9cef9d7 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,115 @@ +#!/bin/sh +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Container entrypoint for ExtendDB. +# +# Argument handling: +# 1. If the first arg is "extenddb", strip it. This lets users copy-paste +# `extenddb ` invocations from docs without surprise: +# docker run extenddb init ... +# docker run init ... +# both work and behave identically. +# 2. If the (remaining) first arg is "serve" or empty (the default CMD), +# run `extenddb serve` and wait on the daemon. +# 3. Otherwise, exec `extenddb "$@"`. Used for init, manage, status, etc. +# +# `extenddb serve` always forks into the background. Without this wrapper +# the container would exit immediately after the parent process returns. +# Once a foreground/no-detach mode lands upstream, this whole script +# collapses to `exec extenddb "$@"`. + +set -eu + +CONFIG="${EXTENDDB_CONFIG:-/etc/extenddb/extenddb.toml}" +STATE_DIR="${HOME:-/var/lib/extenddb}/.extenddb" +RUN_DIR="${STATE_DIR}/run" + +# Strip an optional leading "extenddb" arg so both invocation styles work. +if [ "${1:-}" = "extenddb" ]; then + shift +fi + +# Anything other than "serve" (or no args) goes straight to the binary. +case "${1:-serve}" in + serve) ;; # fall through to the serve path below + *) exec extenddb "$@" ;; +esac + +# --- serve path --- + +if [ ! -f "$CONFIG" ]; then + cat >&2 < init \\ + --config $CONFIG \\ + --pg-host --pg-user --pg-pass \\ + --bind-addr 0.0.0.0 + +See samples/docker/README.md for the full bootstrap walkthrough. +EOF + exit 1 +fi + +extenddb serve --config "$CONFIG" + +# Install the signal-forwarding trap BEFORE waiting for the PID file. +# `extenddb serve` returns to the parent immediately after the double-fork; +# the daemon may take up to ~15 s on a slow runner to write its PID file. +# A SIGTERM landing in that window must still trigger graceful shutdown, +# so the trap tolerates an unset $PID and falls through to the polling +# loop below (which will exit naturally once the daemon is running and +# then dies, or once the wait loop times out). +# +# Why not `tail --pid`? It blocks but does not forward signals. SIGTERM +# to the container would reach `tail`, kill it, and orphan the daemon. +# The container runtime would then fall through to SIGKILL after the +# grace period, skipping graceful shutdown. +PID="" +shutdown() { + if [ -n "${PID:-}" ]; then + echo "extenddb-entrypoint: forwarding ${1:-TERM} to daemon pid $PID" + kill -"${1:-TERM}" "$PID" 2>/dev/null || true + else + echo "extenddb-entrypoint: ${1:-TERM} received before daemon PID was known; exiting" + exit 143 + fi +} +trap 'shutdown TERM' TERM +trap 'shutdown INT' INT + +# extenddb writes a PID file at $run_dir/extenddb-$port.pid. The port comes +# from the config file, which the operator may have changed, so we glob. +# Wait up to ~15 s for the daemon to write its PID. The daemon normally +# writes within ~200 ms but slow CI runners and cold caches can stretch it. +i=0 +while [ "$i" -lt 30 ]; do + PID_FILE="$(ls "${RUN_DIR}"/extenddb-*.pid 2>/dev/null | head -n 1 || true)" + if [ -n "$PID_FILE" ] && [ -f "$PID_FILE" ]; then + break + fi + sleep 0.5 + i=$((i + 1)) +done + +if [ -z "${PID_FILE:-}" ] || [ ! -f "$PID_FILE" ]; then + echo "extenddb-entrypoint: daemon failed to start (no PID file under $RUN_DIR)" >&2 + exit 1 +fi + +PID="$(cat "$PID_FILE")" +echo "extenddb-entrypoint: daemon started (pid $PID, pid-file $PID_FILE)" + +# Poll on the daemon. kill -0 returns 0 while the process is alive. +# 500 ms granularity keeps shutdown responsive without burning CPU. +while kill -0 "$PID" 2>/dev/null; do + sleep 0.5 +done + +echo "extenddb-entrypoint: daemon (pid $PID) exited" diff --git a/docs/manuals/11-deployment-guide.md b/docs/manuals/11-deployment-guide.md index 7b0a2d28..580972c3 100755 --- a/docs/manuals/11-deployment-guide.md +++ b/docs/manuals/11-deployment-guide.md @@ -78,46 +78,58 @@ extenddb init \ ### Containerized -extenddb runs in Docker or Kubernetes. The binary has no runtime dependencies beyond libc and network access to PostgreSQL. - -Example Dockerfile: - -```dockerfile -# Match rust-version in Cargo.toml -FROM rust:1.85 AS builder -WORKDIR /src -COPY . . -RUN cargo build --release - -FROM debian:bookworm-slim -RUN apt-get update && apt-get install -y ca-certificates tini && rm -rf /var/lib/apt/lists/* -COPY --from=builder /src/target/release/extenddb /usr/local/bin/extenddb -COPY extenddb.toml /etc/extenddb/extenddb.toml -COPY entrypoint.sh /usr/local/bin/entrypoint.sh -RUN chmod +x /usr/local/bin/entrypoint.sh -EXPOSE 8000 -ENTRYPOINT ["tini", "--"] -CMD ["/usr/local/bin/entrypoint.sh"] -``` +ExtendDB ships an OCI container image. The image runs `extenddb serve` +only: operators bootstrap the deployment by running `extenddb init` +separately, then mount the resulting `extenddb.toml` as a config volume. + +A `Dockerfile` and a Docker Compose demo live in the repository: -extenddb always daemonizes (there is no foreground mode). In a container, the parent process exits after forking, which causes the container runtime to stop the container. Use `tini` as PID 1 and a wrapper script that starts extenddb and waits on the daemon process: +- [`Dockerfile`](../../Dockerfile) at the repo root: multi-stage build, + `debian:bookworm-slim` runtime, non-root user, tini as PID 1. +- [`samples/docker/compose.yaml`](../../samples/docker/compose.yaml): + PostgreSQL plus an idempotent `extenddb-init` one-shot plus the + long-running `extenddb` service. +- [`samples/docker/bootstrap-iam.sh`](../../samples/docker/bootstrap-iam.sh): + helper script that creates an IAM user with full DynamoDB access and + emits a `extenddb-creds.env` ready to `source`. +- [`samples/docker/README.md`](../../samples/docker/README.md): full + walkthrough including the AWS CLI smoke test. + +For a minute-long evaluation: ```bash -#!/bin/sh -# entrypoint.sh -extenddb serve --config /etc/extenddb/extenddb.toml -# Wait on the daemon PID — the PID file location depends on run_dir in extenddb.toml -# Default: ~/.extenddb/run/extenddb-.pid -PID_FILE="${HOME}/.extenddb/run/extenddb-8000.pid" -if [ -f "$PID_FILE" ]; then - tail --pid="$(cat "$PID_FILE")" -f /dev/null -else - echo "extenddb failed to start — PID file not found at $PID_FILE" >&2 - exit 1 -fi +cd samples/docker +docker compose -f compose.yaml -f compose.dev.yaml up --build -d +./bootstrap-iam.sh +source ./extenddb-creds.env +aws dynamodb list-tables --endpoint-url "$EXTENDDB_ENDPOINT" ``` -For Kubernetes, run `extenddb init` as an init container or a one-time Job, then deploy extenddb as a Deployment with the generated `extenddb.toml` mounted as a ConfigMap or Secret. +For production, do not use the demo `compose.yaml` as-is: passwords +are hard-coded and the cert is self-signed. Use it as a reference and +supply your own values. + +#### Daemonization in containers + +`extenddb serve` always daemonizes. The container entrypoint script +(`docker/entrypoint.sh`) runs `serve`, waits for the daemon's PID file +to appear, then polls the daemon process and forwards SIGTERM/SIGINT +for graceful shutdown. `docker stop` triggers a clean shutdown +(verified to return exit 0 well within the default 10 second grace +period). + +#### Kubernetes + +For Kubernetes deployments: + +- Run `extenddb init` as an `initContainer` or one-time `Job` against a + PersistentVolumeClaim that holds `extenddb.toml`. +- Run `extenddb serve` as the main container in a `Deployment`, + mounting the same PVC at `/etc/extenddb/extenddb.toml` (or use a + ConfigMap / Secret for the rendered config). +- Use a `livenessProbe` / `readinessProbe` against + `https://:8000/health` (with `httpHeaders` skipping TLS + verification, or with the cert mounted from a Secret). ### Air-Gapped diff --git a/samples/docker/README.md b/samples/docker/README.md new file mode 100644 index 00000000..988d8aa7 --- /dev/null +++ b/samples/docker/README.md @@ -0,0 +1,304 @@ +# Run ExtendDB with Docker + +This directory contains a `docker compose` setup that brings up PostgreSQL, +runs `extenddb init` once, then starts ExtendDB on `https://127.0.0.1:8000`. + +It is meant for evaluation and local development. For production deployments, +see [`docs/manuals/11-deployment-guide.md`](../../docs/manuals/11-deployment-guide.md). + +## Prerequisites + +- Docker 25+ with the `docker compose` plugin (or `docker-compose` v2+). +- Free TCP port 8000 on the host (override with `EXTENDDB_HOST_PORT=...`). + +That is the entire prerequisite list. ExtendDB is a single static-ish binary +in the runtime image. PostgreSQL runs as a sibling container; you do not +need a local Postgres install. + +## Quickstart + +```bash +git clone https://github.com/ExtendDB/extenddb.git +cd extenddb/samples/docker + +# Until the image is published to GHCR, build it from source. +docker compose -f compose.yaml -f compose.dev.yaml up --build -d + +# Once published: +# docker compose up -d +``` + +You should see PostgreSQL come up healthy, the `extenddb-init` one-shot +exit cleanly, then `extenddb` start. Verify: + +```bash +curl -k https://127.0.0.1:8000/health +# {"status":"healthy"} +``` + +The DynamoDB API is now live on `https://127.0.0.1:8000`. The web console is +at [`https://127.0.0.1:8000/console/`](https://127.0.0.1:8000/console/) (your +browser will warn about the self-signed certificate; accept it). + +## Make your first DynamoDB request + +ExtendDB requires SigV4 authentication. The compose stack creates an admin +user but no IAM user, so you need one extra step to get usable AWS +credentials. Two paths are documented below. + +### Fast path: bootstrap script + +```bash +./bootstrap-iam.sh # creates user, policy, key + cert +source ./extenddb-creds.env # exports AWS_* env vars +aws dynamodb list-tables --endpoint-url "$EXTENDDB_ENDPOINT" +# {"TableNames": []} +``` + +The script is idempotent (safe to re-run; it bails out if +`extenddb-creds.env` already exists). It writes: + +- `extenddb-creds.env` (mode `0600`): `AWS_ACCESS_KEY_ID`, + `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, `AWS_CA_BUNDLE`, + `EXTENDDB_ENDPOINT`. +- `extenddb-cert.pem`: the server's self-signed certificate, referenced + by `AWS_CA_BUNDLE`. + +Requires `jq` and `docker` on the host. Override the IAM user name with +`EXTENDDB_BOOTSTRAP_USER=...`, the host port with `EXTENDDB_HOST_PORT=...`. + +To re-bootstrap (rotate the access key, refresh the env file): + +```bash +rm extenddb-creds.env +./bootstrap-iam.sh +``` + +### Manual path: what the script does + +If you prefer to understand the steps, run them by hand. The same +result, just verbose. Skip this section if the fast path worked. + +#### 1. Find the default account ID + +The compose stack created a single account during `init`. Look it up: + +```bash +docker compose exec -e EXTENDDB_PASSWORD=admin-local-dev-password \ + extenddb extenddb manage \ + --config /etc/extenddb/extenddb.toml \ + --user admin \ + list-accounts +``` + +You will see one account with an `account_id` like `840625254687`. Note it +down; the rest of the walkthrough uses `$ACCOUNT_ID`. + +```bash +ACCOUNT_ID= +``` + +#### 2. Create an IAM user, attach a policy, and generate an access key + +Create the user: + +```bash +docker compose exec -e EXTENDDB_PASSWORD=admin-local-dev-password \ + extenddb extenddb manage \ + --config /etc/extenddb/extenddb.toml \ + --user admin \ + create-user --account-id "$ACCOUNT_ID" --user-name app +``` + +Attach a policy. ExtendDB enforces IAM authorization on every DynamoDB +request. Without an explicit allow, calls return `AccessDeniedException`. +For evaluation, grant full DynamoDB access: + +```bash +docker compose exec -e EXTENDDB_PASSWORD=admin-local-dev-password \ + extenddb extenddb manage \ + --config /etc/extenddb/extenddb.toml \ + --user admin \ + put-user-policy \ + --account-id "$ACCOUNT_ID" \ + --user-name app \ + --policy-name AllowAllDynamoDB \ + --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"dynamodb:*","Resource":"*"}]}' +``` + +For production, use a least-privilege policy listing only the actions and +resources the application needs. + +Generate an access key: + +```bash +docker compose exec -e EXTENDDB_PASSWORD=admin-local-dev-password \ + extenddb extenddb manage \ + --config /etc/extenddb/extenddb.toml \ + --user admin \ + create-access-key --account-id "$ACCOUNT_ID" --user-name app +``` + +The last command prints an access key ID and secret. Save both. The +secret is shown once. + +```bash +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +export AWS_DEFAULT_REGION=us-east-1 +``` + +#### 3. Trust the self-signed certificate + +Copy the cert out of the container: + +```bash +docker compose cp extenddb:/var/lib/extenddb/.extenddb/tls/cert.pem ./extenddb-cert.pem +export AWS_CA_BUNDLE=$PWD/extenddb-cert.pem +``` + +If you do not want to copy the cert, you can pass `--no-verify-ssl` to the +AWS CLI for every command. The cert approach is safer. + +#### 4. Call the DynamoDB API + +```bash +aws dynamodb list-tables --endpoint-url https://127.0.0.1:8000 +# { "TableNames": [] } + +aws dynamodb create-table \ + --endpoint-url https://127.0.0.1:8000 \ + --table-name greetings \ + --attribute-definitions AttributeName=id,AttributeType=S \ + --key-schema AttributeName=id,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST + +aws dynamodb put-item \ + --endpoint-url https://127.0.0.1:8000 \ + --table-name greetings \ + --item '{"id":{"S":"a"},"msg":{"S":"hello from extenddb"}}' + +aws dynamodb get-item \ + --endpoint-url https://127.0.0.1:8000 \ + --table-name greetings \ + --key '{"id":{"S":"a"}}' +``` + +You should see the item come back. ExtendDB is now running. + +## Common operations + +### Tail server logs + +```bash +docker compose logs -f extenddb +``` + +The first time the server starts you will see lines like: + +``` +extenddb 0.1.0 (catalog 0.0.2) starting on 0.0.0.0:8000 +extenddb server started (pid 10, 0.0.0.0:8000) +extenddb-entrypoint: daemon started (pid 10, pid-file ...) +``` + +### Stop without losing data + +```bash +docker compose stop +``` + +State persists in named volumes. `docker compose start` brings the same +data back. + +### Reset everything + +```bash +docker compose down -v +``` + +This removes containers, networks, **and the named volumes**, including +PostgreSQL data and the ExtendDB config and TLS cert. The next `up` will +re-initialize from scratch. + +### Use a custom host port + +```bash +EXTENDDB_HOST_PORT=8443 docker compose up -d +# now reachable at https://127.0.0.1:8443 +``` + +The container always listens on 8000 internally; this only changes the +host-side mapping. + +### Pin to a specific image tag + +```bash +EXTENDDB_IMAGE=ghcr.io/extenddb/extenddb:v0.1.0 docker compose up -d +``` + +## Going further + +- For a clean walk-through of `init`, `serve`, and credential setup outside + containers, see [`docs/getting-started.md`](../../docs/getting-started.md). +- For production deployment patterns (Kubernetes, RDS/Aurora, multi-arch), + see [`docs/manuals/11-deployment-guide.md`](../../docs/manuals/11-deployment-guide.md). +- For the full set of `extenddb manage` commands, see + [`docs/manuals/05-admin-guide.md`](../../docs/manuals/05-admin-guide.md). + +## Troubleshooting + +### `bind: address already in use` + +Port 8000 is taken on your host. Either stop the conflicting service or +remap: + +```bash +EXTENDDB_HOST_PORT=8080 docker compose up -d +``` + +### `extenddb-init` failed with `Database 'extenddb_catalog' already exists` + +This compose file expects a clean PostgreSQL. If you reused a Postgres +volume from a different ExtendDB install, run: + +```bash +docker compose down -v +docker compose up -d +``` + +### `AccessDeniedException: User ... is not authorized to perform: dynamodb:...` + +The IAM user has an access key but no policy granting DynamoDB actions. +Review step 2: `put-user-policy` must be run before any DynamoDB calls +will succeed. Auto-attached policies cover credential self-service only, +not DynamoDB. + +### `Missing Authentication Token` when calling the DynamoDB API + +The AWS CLI is not signing the request. Confirm you exported +`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_DEFAULT_REGION` +in the same shell, and that the access key was created in step 2 above. + +### `unable to get local issuer certificate` + +The CLI is not trusting the self-signed cert. Either set +`AWS_CA_BUNDLE` to the path of `extenddb-cert.pem` (see step 3), or pass +`--no-verify-ssl` to each `aws dynamodb` invocation. + +### Container exits with `config file not found at /etc/extenddb/extenddb.toml` + +The init step did not run, or its volume was wiped while keeping the +serve volume. Run `docker compose down -v && docker compose up -d` to +restart from a clean state. + +## Security notes for evaluation use + +This compose file is intentionally simple and **not safe for production**: + +- The PostgreSQL admin password is hard-coded in `compose.yaml`. +- The ExtendDB admin password is hard-coded in `compose.yaml`. +- The DynamoDB API uses a self-signed certificate. +- Throttling is off (`throttling_enabled = false` in the generated config). + +For production, follow the production checklist in the deployment guide. diff --git a/samples/docker/bootstrap-iam.sh b/samples/docker/bootstrap-iam.sh new file mode 100755 index 00000000..83006fec --- /dev/null +++ b/samples/docker/bootstrap-iam.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Bootstrap an IAM user with full DynamoDB access against the demo +# compose stack in this directory. After the script succeeds, you can: +# +# source ./extenddb-creds.env +# aws dynamodb list-tables --endpoint-url "$EXTENDDB_ENDPOINT" +# +# Outputs: +# ./extenddb-creds.env -- AWS_* env vars + EXTENDDB_ENDPOINT, ready to source +# ./extenddb-cert.pem -- the server's self-signed certificate +# +# Re-running the script is safe: +# - If extenddb-creds.env already exists, the script exits 0 without +# touching anything. Delete the file to force re-bootstrap. +# - On re-bootstrap, any existing access keys for the IAM user are +# deleted (we cannot recover their secrets) and a fresh key is minted. +# +# Configuration (env vars, all optional): +# EXTENDDB_BOOTSTRAP_USER IAM user name to create (default: app) +# EXTENDDB_BOOTSTRAP_POLICY Inline policy name (default: AllowAllDynamoDB) +# EXTENDDB_ADMIN_USER Admin username (default: admin) +# EXTENDDB_ADMIN_PASSWORD Admin password (default: admin-local-dev-password) +# EXTENDDB_HOST_PORT Host-side port mapping (default: 8000) +# EXTENDDB_COMPOSE Compose file flags (default: -f compose.yaml -f compose.dev.yaml) +# +# Requires: docker (with compose plugin), jq. + +set -euo pipefail + +# --- config --- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="$SCRIPT_DIR/extenddb-creds.env" +CERT_FILE="$SCRIPT_DIR/extenddb-cert.pem" + +USER_NAME="${EXTENDDB_BOOTSTRAP_USER:-app}" +POLICY_NAME="${EXTENDDB_BOOTSTRAP_POLICY:-AllowAllDynamoDB}" +ADMIN_USER="${EXTENDDB_ADMIN_USER:-admin}" +ADMIN_PASSWORD="${EXTENDDB_ADMIN_PASSWORD:-admin-local-dev-password}" +HOST_PORT="${EXTENDDB_HOST_PORT:-8000}" +ENDPOINT="https://127.0.0.1:${HOST_PORT}" +POLICY_DOCUMENT='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"dynamodb:*","Resource":"*"}]}' + +# Compose flags. Word-split intentional to support multiple `-f` flags. +# shellcheck disable=SC2206 +COMPOSE_ARGS=(${EXTENDDB_COMPOSE:--f compose.yaml -f compose.dev.yaml}) + +# --- preflight --- +need() { + command -v "$1" >/dev/null 2>&1 || { + echo "bootstrap-iam: '$1' is required but not installed." >&2 + exit 2 + } +} +need docker +need jq + +cd "$SCRIPT_DIR" + +if [ -f "$ENV_FILE" ]; then + echo "$(basename "$ENV_FILE") already exists. Delete it and re-run to re-bootstrap." + exit 0 +fi + +# Wrapper for `extenddb manage` inside the compose service. +# Pass the admin password through EXTENDDB_PASSWORD rather than the +# --password flag. Inside the container, this keeps the password out of +# the `extenddb manage` argv (where `ps` could see it). On the host, the +# value still appears briefly in `docker compose exec -e ...` argv during +# the call; if that matters in your environment, set EXTENDDB_PASSWORD in +# the script's environment yourself and remove the -e here. +manage() { + docker compose "${COMPOSE_ARGS[@]}" exec -T \ + -e EXTENDDB_PASSWORD="$ADMIN_PASSWORD" \ + extenddb extenddb manage \ + --config /etc/extenddb/extenddb.toml \ + --user "$ADMIN_USER" \ + "$@" +} + +# --- ensure stack is up and healthy --- +if ! docker compose "${COMPOSE_ARGS[@]}" ps --status running --services 2>/dev/null \ + | grep -qx 'extenddb'; then + cat >&2 < Waiting for extenddb to become healthy..." +i=0 +while [ "$i" -lt 60 ]; do # 60 ticks * 1 s = 60 s ceiling + STATUS="$(docker compose "${COMPOSE_ARGS[@]}" ps --format json extenddb 2>/dev/null \ + | jq -r 'if type == "array" then .[0].Health else .Health end // "unknown"')" + case "$STATUS" in + healthy) + echo " healthy." + break + ;; + starting|unknown|"") + sleep 1 + ;; + *) + echo "bootstrap-iam: extenddb container reported unhealthy state: $STATUS" >&2 + exit 1 + ;; + esac + i=$((i + 1)) +done + +if [ "$STATUS" != "healthy" ]; then + echo "bootstrap-iam: extenddb did not become healthy within 60 s (last status: $STATUS)." >&2 + echo "bootstrap-iam: check 'docker compose ${COMPOSE_ARGS[*]} logs extenddb'." >&2 + exit 1 +fi + +# --- discover account --- +echo "==> Discovering default account..." +ACCOUNT_ID="$(manage list-accounts | jq -r '.[0].account_id // empty')" +if [ -z "$ACCOUNT_ID" ]; then + echo "bootstrap-iam: no account found. Was 'extenddb init' run?" >&2 + exit 1 +fi +echo " account_id=$ACCOUNT_ID" + +# --- ensure IAM user --- +echo "==> Ensuring IAM user '$USER_NAME' exists..." +if manage list-users --account-id "$ACCOUNT_ID" \ + | jq -e --arg u "$USER_NAME" '.[] | select(.user_name == $u)' >/dev/null; then + echo " user already exists." +else + manage create-user --account-id "$ACCOUNT_ID" --user-name "$USER_NAME" >/dev/null + echo " user created." +fi + +# --- ensure policy --- +echo "==> Ensuring policy '$POLICY_NAME' is attached..." +if manage list-user-policies --account-id "$ACCOUNT_ID" --user-name "$USER_NAME" \ + | jq -e --arg p "$POLICY_NAME" '.[] | select(.policy_name == $p)' >/dev/null; then + echo " policy already attached." +else + manage put-user-policy \ + --account-id "$ACCOUNT_ID" \ + --user-name "$USER_NAME" \ + --policy-name "$POLICY_NAME" \ + --policy-document "$POLICY_DOCUMENT" >/dev/null + echo " policy attached." +fi + +echo "==> Minting access key..." +EXISTING_KEYS="$(manage list-access-keys --account-id "$ACCOUNT_ID" --user-name "$USER_NAME" \ + | jq -r '.[].access_key_id // empty')" +if [ -n "$EXISTING_KEYS" ]; then + while IFS= read -r KEY_ID; do + [ -z "$KEY_ID" ] && continue + echo " deleting stale key $KEY_ID..." + manage delete-access-key \ + --account-id "$ACCOUNT_ID" \ + --user-name "$USER_NAME" \ + --access-key-id "$KEY_ID" >/dev/null + done <<< "$EXISTING_KEYS" +fi + +KEY_JSON="$(manage create-access-key --account-id "$ACCOUNT_ID" --user-name "$USER_NAME")" +ACCESS_KEY_ID="$(echo "$KEY_JSON" | jq -r '.access_key_id')" +SECRET_ACCESS_KEY="$(echo "$KEY_JSON" | jq -r '.secret_access_key')" +if [ -z "$ACCESS_KEY_ID" ] || [ -z "$SECRET_ACCESS_KEY" ] \ + || [ "$ACCESS_KEY_ID" = "null" ] || [ "$SECRET_ACCESS_KEY" = "null" ]; then + echo "bootstrap-iam: failed to parse access key from create-access-key output:" >&2 + echo "$KEY_JSON" >&2 + exit 1 +fi +echo " new key: $ACCESS_KEY_ID" + +# --- export cert --- +echo "==> Exporting TLS certificate..." +docker compose "${COMPOSE_ARGS[@]}" cp \ + extenddb:/var/lib/extenddb/.extenddb/tls/cert.pem "$CERT_FILE" >/dev/null +chmod 0644 "$CERT_FILE" + +# --- write env file --- +umask 0177 # ensure new file is 0600 +cat > "$ENV_FILE" <