From 28ea2b5b5dd43614ceb313aedba84c18fa3d3a2f Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 4 Jun 2026 15:13:30 -0400 Subject: [PATCH 1/7] docs(config): add annotated default config templates for MySQL and Spanner Provide ready-to-edit, fully-annotated example configs for the two standard builds. local.example.toml is now a documented MySQL defaut local.example.spanner.toml is a new template for the production Spanner syncstorage + MySQL tokenserver. Each setting notes its purpose, default, and backend relevance, and the REQUIRED values a user must edit are called out explicitly. Closes STOR-477 --- config/local.example.spanner.toml | 101 ++++++++++++++++++++++++++++++ config/local.example.toml | 95 +++++++++++++++++++++++++--- 2 files changed, 187 insertions(+), 9 deletions(-) create mode 100644 config/local.example.spanner.toml diff --git a/config/local.example.spanner.toml b/config/local.example.spanner.toml new file mode 100644 index 0000000000..34ab6488c8 --- /dev/null +++ b/config/local.example.spanner.toml @@ -0,0 +1,101 @@ +# ============================================================================= +# Default out-of-the-box configuration: Spanner build +# ============================================================================= +# +# This is a ready-to-edit template for running syncstorage-rs against the +# standard Spanner build (`make run_spanner`, the `syncstorage-rs-spanner` +# Docker image). This mirrors the production topology: Syncstorage runs against +# Google Cloud Spanner, while Tokenserver runs against MySQL. +# +# Copy it to `config/local.toml` and edit the values marked REQUIRED: +# +# cp config/local.example.spanner.toml config/local.toml +# +# Every setting can also be supplied as an environment variable prefixed with +# `SYNC_` (nested keys use `__`), e.g. `syncstorage.database_url` becomes +# `SYNC_SYNCSTORAGE__DATABASE_URL`. Environment variables take precedence over +# the config file. See the full reference at docs/src/config.md, and the +# annotated `Settings` structs in `*-settings/src/lib.rs`. +# +# To run against the local Spanner emulator instead of real Spanner, set +# `syncstorage.spanner_emulator_host` (see below) and point the +# `GOOGLE_APPLICATION_CREDENTIALS` env var at a service-account key for real +# Spanner only. +# +# For the all-MySQL build, see `config/local.example.toml`. +# ============================================================================= + +# REQUIRED. Secret used to derive the Hawk signing/token secrets. Use a long, +# random string in any real deployment. No default. +master_secret = "INSERT_SECRET_KEY_HERE" + +# Host/port the server binds to. Defaults: host = "127.0.0.1", port = 8000. +# Use "0.0.0.0" inside containers so the port is reachable from the host. +host = "127.0.0.1" +port = 8000 + +# Emit human-readable logs instead of mozlog JSON. Default: false (0). +human_logs = 1 + +# ----------------------------------------------------------------------------- +# Syncstorage (BSO storage service) — Spanner +# ----------------------------------------------------------------------------- + +# Enable the Syncstorage service. Default: true. +syncstorage.enabled = true + +# REQUIRED when syncstorage is enabled. Spanner DSN, of the form +# spanner://projects//instances//databases/. +# No usable default; the server fails fast at startup if it is unset. A +# "spanner://" scheme is what switches the syncstorage backend into Spanner +# mode (see `Settings::uses_spanner`). +syncstorage.database_url = "spanner://projects/SAMPLE_GCP_PROJECT/instances/SAMPLE_SPANNER_INSTANCE/databases/SAMPLE_SPANNER_DB" + +# Local Spanner emulator host. Leave unset for real Spanner. When running the +# emulator (e.g. via docker/docker-compose.spanner.yaml) set this to the +# emulator's gRPC address so the client talks to the emulator instead of GCP. +# syncstorage.spanner_emulator_host = "localhost:9010" + +# Max size of the syncstorage DB connection pool. Default: 10. +# syncstorage.database_pool_max_size = 10 + +# Quota tracking/enforcement (Spanner-only). Both default to false. Enable +# tracking to record per-user usage; additionally enable enforcement to reject +# writes over `limits.max_quota_limit` (default 2 GB). +syncstorage.enable_quota = 0 +# syncstorage.enforce_quota = 0 + +# ----------------------------------------------------------------------------- +# Tokenserver (node assignment + FxA OAuth verification) — MySQL +# ----------------------------------------------------------------------------- + +# Enable Tokenserver. Default: false. +tokenserver.enabled = true + +# REQUIRED when tokenserver is enabled. In the production topology Tokenserver +# uses MySQL even when Syncstorage uses Spanner. No usable default. +tokenserver.database_url = "mysql://sample_user:sample_password@localhost/tokenserver_rs" + +# Backend type reported in the token response (telemetry only). For the Spanner +# topology leave this as the default "spanner". Valid: "mysql"/"postgres"/"spanner". +tokenserver.node_type = "spanner" + +# Run the tokenserver DB migrations on startup. Default: false. +# tokenserver.run_migrations = true + +# FxA environment used to verify OAuth tokens. The defaults below point at the +# FxA *stage* environment; point them at production for a real deployment +# (fxa_email_domain = "api.accounts.firefox.com", +# fxa_oauth_server_url = "https://oauth.accounts.firefox.com"). +tokenserver.fxa_email_domain = "api-accounts.stage.mozaws.net" +tokenserver.fxa_oauth_server_url = "https://oauth.stage.mozaws.net" + +# REQUIRED for real deployments. Secret used to hash users' metrics UIDs so +# they stay anonymous. Default: "secret" (fine for local dev only). +tokenserver.fxa_metrics_hash_secret = "INSERT_SECRET_KEY_HERE" + +# ----------------------------------------------------------------------------- +# CORS (optional; sensible defaults applied when unset) +# ----------------------------------------------------------------------------- +# cors_allowed_origin = "localhost" # Default: "*" +# cors_max_age = 86400 # Default: 1728000 (20 days) diff --git a/config/local.example.toml b/config/local.example.toml index 6a19cf5c1e..6961c08614 100644 --- a/config/local.example.toml +++ b/config/local.example.toml @@ -1,6 +1,37 @@ +# ============================================================================= +# Default out-of-the-box configuration: MySQL build +# ============================================================================= +# +# This is a ready-to-edit template for running syncstorage-rs against the +# standard MySQL build (the default `cargo build` / `make run_mysql` target, +# and the `syncstorage-rs-mysql` Docker image). It runs both Syncstorage and +# Tokenserver against MySQL. +# +# Copy it to `config/local.toml` and edit the values marked REQUIRED: +# +# cp config/local.example.toml config/local.toml +# +# Every setting can also be supplied as an environment variable prefixed with +# `SYNC_` (nested keys use `__`), e.g. `syncstorage.database_url` becomes +# `SYNC_SYNCSTORAGE__DATABASE_URL`. Environment variables take precedence over +# the config file. See the full reference at docs/src/config.md, and the +# annotated `Settings` structs in `*-settings/src/lib.rs`. +# +# For the Spanner build, see `config/local.example.spanner.toml`. +# ============================================================================= + +# REQUIRED. Secret used to derive the Hawk signing/token secrets. Use a long, +# random string in any real deployment. No default; the server cannot +# authenticate requests without it. master_secret = "INSERT_SECRET_KEY_HERE" -# removing this line will default to moz_json formatted logs (which is preferred for production envs) +# Host/port the server binds to. Defaults: host = "127.0.0.1", port = 8000. +# Use "0.0.0.0" inside containers so the port is reachable from the host. +host = "127.0.0.1" +port = 8000 + +# Emit human-readable logs instead of mozlog JSON. Default: false (0). +# Leave JSON logging on for production; it is the format our tooling expects. human_logs = 1 # Example Syncstorage settings: @@ -16,16 +47,62 @@ syncstorage.enable_quota = 0 # max_quota_limit = 200000000 syncstorage.enabled = true syncstorage.limits.max_total_records = 9984 +# ----------------------------------------------------------------------------- +# Syncstorage (BSO storage service) +# ----------------------------------------------------------------------------- -# Example Tokenserver settings: -# database_url is required when tokenserver is enabled; there is no default, -# and the server fails to start if it is unset. -tokenserver.database_url = "mysql://sample_user:sample_password@localhost/tokenserver_rs" +# Enable the Syncstorage service. Default: true. +syncstorage.enabled = true + +# REQUIRED when syncstorage is enabled. MySQL DSN for the syncstorage database. +# There is no usable default: the server fails fast at startup if it is unset, +# rather than silently connecting to a default host. +syncstorage.database_url = "mysql://sample_user:sample_password@localhost/syncstorage_rs" + +# Max size of the syncstorage DB connection pool. Default: 10. +# syncstorage.database_pool_max_size = 10 + +# Quota tracking/enforcement are Spanner-only features and are force-disabled +# for non-Spanner backends, so they have no effect on the MySQL build. +syncstorage.enable_quota = 0 + +# Request/batch limits. Defaults shown in docs/src/config.md (Syncstorage +# Limits). `max_total_records` is also used to cap MySQL `get_bsos` result +# sizes; see issues #298/#333. +syncstorage.limits.max_total_records = 1666 + +# ----------------------------------------------------------------------------- +# Tokenserver (node assignment + FxA OAuth verification) +# ----------------------------------------------------------------------------- + +# Enable Tokenserver. Default: false. Enable it for a self-contained server +# that both assigns nodes and stores data. tokenserver.enabled = true + +# REQUIRED when tokenserver is enabled. MySQL DSN for the tokenserver database. +# No usable default; the server fails fast at startup if it is unset. +tokenserver.database_url = "mysql://sample_user:sample_password@localhost/tokenserver_rs" + +# Backend type reported in the token response (telemetry only). For the MySQL +# build set this to "mysql". Default: "spanner". Valid: "mysql"/"postgres"/"spanner". +tokenserver.node_type = "mysql" + +# Run the tokenserver DB migrations on startup. Default: false. +# tokenserver.run_migrations = true + +# FxA environment used to verify OAuth tokens. The defaults below point at the +# FxA *stage* environment; point them at production for a real deployment +# (fxa_email_domain = "api.accounts.firefox.com", +# fxa_oauth_server_url = "https://oauth.accounts.firefox.com"). tokenserver.fxa_email_domain = "api-accounts.stage.mozaws.net" -tokenserver.fxa_metrics_hash_secret = "INSERT_SECRET_KEY_HERE" tokenserver.fxa_oauth_server_url = "https://oauth.stage.mozaws.net" -# cors settings -# cors_allowed_origin = "localhost" -# cors_max_age = 86400 +# REQUIRED for real deployments. Secret used to hash users' metrics UIDs so +# they stay anonymous. Default: "secret" (fine for local dev only). +tokenserver.fxa_metrics_hash_secret = "INSERT_SECRET_KEY_HERE" + +# ----------------------------------------------------------------------------- +# CORS (optional; sensible defaults applied when unset) +# ----------------------------------------------------------------------------- +# cors_allowed_origin = "localhost" # Default: "*" +# cors_max_age = 86400 # Default: 1728000 (20 days) From a0cac16b1714dc615ba2c7151bcc795c2a556d3c Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 4 Jun 2026 15:13:30 -0400 Subject: [PATCH 2/7] docs: annotate settings struct fields with defaults and usage Add doc comments to the previously-undocumented public Settings fields (host/port, statsd host/port, human_logs, pool size, quota toggles, spanner_emulator_host, enabled, statsd_label) so the structs are the source of truth for configuration defaults. Documentation only; no behavior change. Closes STOR-477 --- syncserver-settings/src/lib.rs | 7 +++++++ syncstorage-settings/src/lib.rs | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/syncserver-settings/src/lib.rs b/syncserver-settings/src/lib.rs index bbdf2ebd5c..1feb487c11 100644 --- a/syncserver-settings/src/lib.rs +++ b/syncserver-settings/src/lib.rs @@ -16,7 +16,10 @@ static PREFIX: &str = "sync"; #[derive(Clone, Debug, Deserialize)] #[serde(default)] pub struct Settings { + /// TCP port the server binds to. Default: 8000. pub port: u16, + /// Host address the server binds to. Default: "127.0.0.1". Use "0.0.0.0" + /// inside containers so the port is reachable from the host. pub host: String, /// Keep-alive header value (seconds) pub actix_keep_alive: Option, @@ -25,9 +28,13 @@ pub struct Settings { /// that are used during Hawk authentication. pub master_secret: Secrets, + /// Emit human-readable logs instead of mozlog JSON. Default: false. + /// Production environments should leave this off (JSON is preferred). pub human_logs: bool, + /// Hostname of the StatsD/metrics sink. Default: "localhost". pub statsd_host: Option, + /// Port of the StatsD/metrics sink. Default: 8125. pub statsd_port: u16, /// Whether to include the hostname in metrics, which increases cardinality significantly in /// prod. diff --git a/syncstorage-settings/src/lib.rs b/syncstorage-settings/src/lib.rs index 76e2171add..10d7bcb7d5 100644 --- a/syncstorage-settings/src/lib.rs +++ b/syncstorage-settings/src/lib.rs @@ -75,6 +75,7 @@ pub struct Settings { /// fails fast at startup (see `syncserver_settings::Settings::validate`) /// rather than silently connecting to a default host. pub database_url: String, + /// Max size of the database connection pool. Default: 10. pub database_pool_max_size: u32, /// Pool timeout when waiting for a slot to become available, in seconds pub database_pool_connection_timeout: Option, @@ -92,15 +93,24 @@ pub struct Settings { /// Server-enforced limits for request payloads. pub limits: ServerLimits, + /// StatsD metrics label prefix for syncstorage. Default: "syncstorage". pub statsd_label: String, + /// Track per-user storage quota. Spanner-only; force-disabled on other + /// backends. Default: false. pub enable_quota: bool, + /// Reject writes that exceed `limits.max_quota_limit`. Requires + /// `enable_quota`. Spanner-only; force-disabled on other backends. + /// Default: false. pub enforce_quota: bool, /// Whether Glean telemetry metric emission is enabled. pub glean_enabled: bool, + /// gRPC address of a local Spanner emulator (e.g. "localhost:9010"). + /// Leave unset to use real Spanner. Default: None. pub spanner_emulator_host: Option, + /// Whether the Syncstorage service is enabled. Default: true. pub enabled: bool, /// Fail the `/__lbheartbeat__` healthcheck after running for this duration From 0c14a86cb4d32272e0da1c59d78a9ab54684994f Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 4 Jun 2026 15:13:30 -0400 Subject: [PATCH 3/7] docs: add default Spanner/MySQL config guide and one-shot MySQL compose Add a short How-To page describing the out-of-the-box configuration for the standard MySQL and Spanner builds, the minimum required settings, and where the full reference lives. Add a one-shot MySQL docker compose recipe (DB + server, migrations applied automatically, sync-1.5 service + node bootstrapped) so the stack comes up ready to serve. The primary recipe builds the image from source (works from a checkout with no published tag); an alternative uses a published image, noting that images are tagged by commit SHA with no `latest` tag, so SYNCSERVER_VERSION must be pinned. Register the new page in the book summary and How-To index. Closes STOR-477 --- docs/src/SUMMARY.md | 1 + docs/src/how-to/default-config.md | 68 +++++++++++ docs/src/how-to/how-to-run-with-docker.md | 141 +++++++++++++++++++++- docs/src/how-to/index.md | 1 + 4 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 docs/src/how-to/default-config.md diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index a0e830afdd..777f9259db 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -37,6 +37,7 @@ [Mozilla Accounts Server - FxA](mozilla-accounts.md) - [How To Guides](how-to/index.md) + - [Default Configuration for Spanner & MySQL Builds](how-to/default-config.md) - [Run Your Own Sync-1.5 Server with Docker](how-to/how-to-run-with-docker.md) - [Run Your Own Sync-1.5 Server (legacy)](how-to/how-to-run-sync-server.md) - [Configure Sync Server for TLS (legacy)](how-to/how-to-config-tls.md) diff --git a/docs/src/how-to/default-config.md b/docs/src/how-to/default-config.md new file mode 100644 index 0000000000..4d5dde7392 --- /dev/null +++ b/docs/src/how-to/default-config.md @@ -0,0 +1,68 @@ +# Default Configuration for Spanner & MySQL Builds + +This page describes the out-of-the-box configuration needed to run the two +standard `syncstorage-rs` builds: + +- **MySQL build** — the default `cargo build` / `make run_mysql` target and the + `syncstorage-rs-mysql` Docker image. Both Syncstorage and Tokenserver run + against MySQL. +- **Spanner build** — `make run_spanner` and the `syncstorage-rs-spanner` + Docker image. This mirrors production: Syncstorage runs against Google Cloud + Spanner while Tokenserver runs against MySQL. + +Annotated, copy-paste-ready templates live in the repo: + +| Build | Template | +| --- | --- | +| MySQL | [`config/local.example.toml`](https://github.com/mozilla-services/syncstorage-rs/blob/master/config/local.example.toml) | +| Spanner | [`config/local.example.spanner.toml`](https://github.com/mozilla-services/syncstorage-rs/blob/master/config/local.example.spanner.toml) | + +Copy one to `config/local.toml` and edit the values marked `REQUIRED`: + +```sh +cp config/local.example.toml config/local.toml # MySQL +# or +cp config/local.example.spanner.toml config/local.toml # Spanner +``` + +Every setting can also be supplied as an environment variable prefixed with +`SYNC_` (nested keys use `__`). For example `syncstorage.database_url` becomes +`SYNC_SYNCSTORAGE__DATABASE_URL`. Environment variables take precedence over the +config file. The complete list of options and their defaults is in the +[Application Configuration](../config.md) reference, and the source of truth is +the doc-commented `Settings` structs in the `*-settings` crates. + +## Minimum required settings + +Most settings have sensible defaults. Regardless of backend you must supply: + +| Setting | Why | +| --- | --- | +| [`master_secret`](../config.md#SYNC_MASTER_SECRET) | Derives the Hawk signing/token secrets. No default. | +| [`syncstorage.database_url`](../config.md#SYNC_SYNCSTORAGE__DATABASE_URL) | No usable default; the server fails fast at startup if unset. | +| [`tokenserver.database_url`](../config.md#SYNC_TOKENSERVER__DATABASE_URL) | Required when `tokenserver.enabled = true` (fails fast if unset). | + +Backend-specific notes: + +- **MySQL:** set `tokenserver.node_type = "mysql"`. Syncstorage MySQL schema + migrations run automatically at startup; set + `tokenserver.run_migrations = true` to apply the Tokenserver schema too. +- **Spanner:** `syncstorage.database_url` must use the + `spanner://projects/.../instances/.../databases/...` form — the `spanner://` + scheme is what selects the Spanner backend. Leave `tokenserver.node_type` at + its default (`"spanner"`). To run against the local emulator instead of GCP, + set `syncstorage.spanner_emulator_host` (e.g. `localhost:9010`); for real + Spanner, point `GOOGLE_APPLICATION_CREDENTIALS` at a service-account key. + +## Run it out of the box with Docker + +For a zero-to-running MySQL stack (database + server, schema applied +automatically, ready to serve), see the +[one-shot MySQL `docker compose`](how-to-run-with-docker.md#docker-compose-one-shot-with-mysql) +recipe. It brings up MySQL and the server, runs migrations, and bootstraps the +storage node so that `curl http://localhost:8000/__heartbeat__` succeeds with no +further setup. + +A Spanner stack cannot be fully zero-config: real Spanner requires GCP +credentials, and the local emulator requires extra wiring (see +`docker/docker-compose.spanner.yaml` and `make run_spanner`). diff --git a/docs/src/how-to/how-to-run-with-docker.md b/docs/src/how-to/how-to-run-with-docker.md index b335189b7f..833661d8e6 100644 --- a/docs/src/how-to/how-to-run-with-docker.md +++ b/docs/src/how-to/how-to-run-with-docker.md @@ -10,7 +10,7 @@ Images are available for both and [PostgreSQL](https://github.com/mozilla-services/syncstorage-rs/pkgs/container/syncstorage-rs%2Fsyncstorage-rs-postgres) as the database. Differences in configuration or deployment steps will be -noted. +noted. Tagged release builds are available on ghcr.io. To pin to a specific version, set `SYNCSERVER_VERSION` to the desired release tag (e.g., `SYNCSERVER_VERSION=v1.45.0`) @@ -142,6 +142,145 @@ Next, start the service with `docker compose`: docker compose -f docker-compose.one-shot.yaml up -d ``` +## Docker Compose, One-Shot with MySQL + +This recipe brings up everything needed for a working MySQL-backed server in a +single command: a MySQL database for Syncstorage and one for Tokenserver, plus +the server itself. Syncstorage applies its schema migrations automatically at +startup, `SYNC_TOKENSERVER__RUN_MIGRATIONS` applies the Tokenserver schema, and +`SYNC_TOKENSERVER__INIT_NODE_URL` bootstraps the `sync-1.5` service and storage +node records — so the stack is ready to serve with no manual database setup. + +### Option A: build from source (works from a checkout) + +Run this from a clone of the repository; the build `context` is the repo root, +so the MySQL build of the server is compiled locally and the recipe does not +depend on any published image. Save the yaml below into a file, e.g. +`docker-compose.one-shot.yaml`. + +```yaml +services: + syncserver: + build: + context: . + args: + SYNCSTORAGE_DATABASE_BACKEND: mysql + TOKENSERVER_DATABASE_BACKEND: mysql + container_name: syncserver + ports: + - "8000:8000" + environment: + SYNC_HOST: "0.0.0.0" + SYNC_PORT: "8000" + SYNC_MASTER_SECRET: "${SYNC_MASTER_SECRET:-changeme_secret_key}" + SYNC_SYNCSTORAGE__DATABASE_URL: "mysql://sync:sync@sync-db:3306/syncstorage" + SYNC_TOKENSERVER__DATABASE_URL: "mysql://sync:sync@tokenserver-db:3306/tokenserver" + SYNC_TOKENSERVER__ENABLED: "true" + SYNC_TOKENSERVER__NODE_TYPE: "mysql" + SYNC_TOKENSERVER__RUN_MIGRATIONS: "true" + SYNC_TOKENSERVER__FXA_EMAIL_DOMAIN: "api.accounts.firefox.com" + SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL: "https://oauth.accounts.firefox.com" + SYNC_HUMAN_LOGS: "${SYNC_HUMAN_LOGS:-false}" + RUST_LOG: "${RUST_LOG:-info}" + SYNC_TOKENSERVER__INIT_NODE_URL: "${SYNC_TOKENSERVER__INIT_NODE_URL:-http://localhost:${SYNC_PORT:-8000}}" + depends_on: + sync-db: + condition: service_healthy + tokenserver-db: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/__heartbeat__"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + sync-db: + image: docker.io/library/mysql:8.0 + container_name: syncserver-sync-db + command: --explicit_defaults_for_timestamp + environment: + MYSQL_RANDOM_ROOT_PASSWORD: "yes" + MYSQL_DATABASE: syncstorage + MYSQL_USER: sync + MYSQL_PASSWORD: sync + volumes: + - sync_db_data:/var/lib/mysql + healthcheck: + test: ["CMD-SHELL", "mysqladmin -h 127.0.0.1 -usync -psync ping"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + + tokenserver-db: + image: docker.io/library/mysql:8.0 + container_name: syncserver-tokenserver-db + command: --explicit_defaults_for_timestamp + environment: + MYSQL_RANDOM_ROOT_PASSWORD: "yes" + MYSQL_DATABASE: tokenserver + MYSQL_USER: sync + MYSQL_PASSWORD: sync + volumes: + - tokenserver_db_data:/var/lib/mysql + healthcheck: + test: ["CMD-SHELL", "mysqladmin -h 127.0.0.1 -usync -psync ping"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + +volumes: + sync_db_data: + driver: local + tokenserver_db_data: + driver: local +``` + +Next, build and start the service with `docker compose`: + +```sh +docker compose -f docker-compose.one-shot.yaml up -d --build +``` + +Once the `syncserver` container reports healthy, confirm it is serving: + +```sh +curl http://localhost:8000/__heartbeat__ +``` + +### Option B: use a published image + +Mozilla also publishes prebuilt images on ghcr.io. Note that these are +currently tagged by commit SHA — there is **no `latest` or semver tag** — so +you must pin `SYNCSERVER_VERSION` to a tag listed on the +[`syncstorage-rs-mysql` packages page](https://github.com/mozilla-services/syncstorage-rs/pkgs/container/syncstorage-rs%2Fsyncstorage-rs-mysql). +To use a published image, replace the `syncserver` service's `build:` block with +an `image:` reference; the `sync-db`, `tokenserver-db`, and `volumes` sections +are unchanged: + +```yaml +services: + syncserver: + image: ghcr.io/mozilla-services/syncstorage-rs/syncstorage-rs-mysql:${SYNCSERVER_VERSION:?set SYNCSERVER_VERSION to a published tag} + platform: linux/amd64 + container_name: syncserver + # ...the remaining syncserver settings are identical to Option A +``` + +Then start it with the tag pinned (published images are `linux/amd64`): + +```sh +SYNCSERVER_VERSION= docker compose -f docker-compose.one-shot.yaml up -d +``` + +> Set `SYNC_MASTER_SECRET` to your own value for anything beyond local +> experimentation; the default above is a placeholder. + ## Configuring Firefox (Desktop) Firefox itself needs to be configured to use the self-hosted Sync server. diff --git a/docs/src/how-to/index.md b/docs/src/how-to/index.md index 07de25c9a8..c35186c16d 100644 --- a/docs/src/how-to/index.md +++ b/docs/src/how-to/index.md @@ -2,6 +2,7 @@ Collection of How To guides for various Sync-related operations. +- [Default Configuration for Spanner & MySQL Builds](default-config.md) - [Run Your Own Sync-1.5 Server with Docker](how-to-run-with-docker.md) - [Run Your Own Sync-1.5 Server (legacy)](how-to-run-sync-server.md) - [Configure Sync Server for TLS (legacy)](how-to-config-tls.md) From d333b402b6a6270372c9484baa02f509564b6d18 Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 18 Jun 2026 17:15:02 -0400 Subject: [PATCH 4/7] docs(config): de-duplicate keys in local.example.toml A merge left a stale settings block in the MySQL example config that re-declared syncstorage.enabled/database_url/enable_quota and limits.max_total_records, producing duplicate TOML keys and a parse failure (`Cannot overwrite a value`). Remove the stale block, keeping the single annotated definitions so the template parses and can be copied to local.toml. Closes STOR-477 --- config/local.example.toml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/config/local.example.toml b/config/local.example.toml index 6961c08614..94dbda769a 100644 --- a/config/local.example.toml +++ b/config/local.example.toml @@ -34,19 +34,6 @@ port = 8000 # Leave JSON logging on for production; it is the format our tooling expects. human_logs = 1 -# Example Syncstorage settings: -# database_url is required when syncstorage is enabled; there is no default, -# and the server fails to start if it is unset. -# Example MySQL DSN: -syncstorage.database_url = "mysql://sample_user:sample_password@localhost/syncstorage_rs" -# Example Spanner DSN: -# syncstorage.database_url="spanner://projects/SAMPLE_GCP_PROJECT/instances/SAMPLE_SPANNER_INSTANCE/databases/SAMPLE_SPANNER_DB" -# enable quota limits -syncstorage.enable_quota = 0 -# set the quota limit to 2GB. -# max_quota_limit = 200000000 -syncstorage.enabled = true -syncstorage.limits.max_total_records = 9984 # ----------------------------------------------------------------------------- # Syncstorage (BSO storage service) # ----------------------------------------------------------------------------- From 099196743298fd8ef21b7ebc9a75dc3f56a549ed Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 18 Jun 2026 17:17:21 -0400 Subject: [PATCH 5/7] docs(config): set MySQL example max_total_records to 9984 Use 9984 as the example syncstorage.limits.max_total_records value. Closes STOR-477 --- config/local.example.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/local.example.toml b/config/local.example.toml index 94dbda769a..2318a6d2bb 100644 --- a/config/local.example.toml +++ b/config/local.example.toml @@ -56,7 +56,7 @@ syncstorage.enable_quota = 0 # Request/batch limits. Defaults shown in docs/src/config.md (Syncstorage # Limits). `max_total_records` is also used to cap MySQL `get_bsos` result # sizes; see issues #298/#333. -syncstorage.limits.max_total_records = 1666 +syncstorage.limits.max_total_records = 9984 # ----------------------------------------------------------------------------- # Tokenserver (node assignment + FxA OAuth verification) From 3abc6aa1e2131dd7764deb4b3d97c2f4f3b708f7 Mon Sep 17 00:00:00 2001 From: Taddes Date: Tue, 23 Jun 2026 13:03:30 -0400 Subject: [PATCH 6/7] chore(docker): add one-shot stand-alone compose files and make targets The one-shot install recipes previously existed only as copy-paste YAML in the docs, so the out-of-box install required pasting YAML into a file before running docker compose. Promote them to committed, build-from-source compose files for MySQL, PostgreSQL, and the Spanner emulator, each with a make target (docker_oneshot_ / _stop), so a working server comes up in a single command with no manual database setup. Consolidate the docs to reference the committed files instead of duplicating the YAML, keeping the compose files as the single source of truth. Closes STOR-477 --- Makefile | 18 ++ docker/docker-compose.one-shot.mysql.yaml | 112 +++++++++ docker/docker-compose.one-shot.postgres.yaml | 85 +++++++ docker/docker-compose.one-shot.spanner.yaml | 113 ++++++++++ docs/src/how-to/how-to-run-with-docker.md | 226 ++++--------------- 5 files changed, 375 insertions(+), 179 deletions(-) create mode 100644 docker/docker-compose.one-shot.mysql.yaml create mode 100644 docker/docker-compose.one-shot.postgres.yaml create mode 100644 docker/docker-compose.one-shot.spanner.yaml diff --git a/Makefile b/Makefile index 8bc5d4e0d7..30d9cde48b 100644 --- a/Makefile +++ b/Makefile @@ -102,6 +102,24 @@ docker_start_spanner_rebuild: docker_stop_spanner: docker compose -f docker/docker-compose.spanner.yaml down +docker_oneshot_mysql: ## Build & run a stand-alone MySQL Syncserver (curl localhost:8000/__heartbeat__). + docker compose -f docker/docker-compose.one-shot.mysql.yaml up -d --build + +docker_oneshot_mysql_stop: ## Stop the stand-alone MySQL Syncserver. + docker compose -f docker/docker-compose.one-shot.mysql.yaml down + +docker_oneshot_postgres: ## Build & run a stand-alone PostgreSQL Syncserver (curl localhost:8000/__heartbeat__). + docker compose -f docker/docker-compose.one-shot.postgres.yaml up -d --build + +docker_oneshot_postgres_stop: ## Stop the stand-alone PostgreSQL Syncserver. + docker compose -f docker/docker-compose.one-shot.postgres.yaml down + +docker_oneshot_spanner: ## Build & run a stand-alone Spanner-emulator Syncserver, local dev only (curl localhost:8000/__heartbeat__). + docker compose -f docker/docker-compose.one-shot.spanner.yaml up -d --build + +docker_oneshot_spanner_stop: ## Stop the stand-alone Spanner-emulator Syncserver. + docker compose -f docker/docker-compose.one-shot.spanner.yaml down + .ONESHELL: docker_run_mysql_e2e_tests: exit_code=0 diff --git a/docker/docker-compose.one-shot.mysql.yaml b/docker/docker-compose.one-shot.mysql.yaml new file mode 100644 index 0000000000..5f6320b122 --- /dev/null +++ b/docker/docker-compose.one-shot.mysql.yaml @@ -0,0 +1,112 @@ +# One-shot, stand-alone MySQL-backed Syncserver. +# +# Unlike docker-compose.mysql.yaml (which is an end-to-end test harness: +# mock FxA server, prebuilt `app:build` image), this brings up a working +# server in a single command with no manual database setup: +# +# make docker_oneshot_mysql +# # or: +# docker compose -f docker/docker-compose.one-shot.mysql.yaml up -d --build +# +# Then confirm it is serving: +# +# curl http://localhost:8000/__heartbeat__ +# +# Syncstorage applies its schema migrations at startup, +# SYNC_TOKENSERVER__RUN_MIGRATIONS applies the Tokenserver schema, and +# SYNC_TOKENSERVER__INIT_NODE_URL bootstraps the `sync-1.5` service and storage +# node records. +# +# This builds the MySQL server from the local checkout (build context is the +# repo root), so it does not depend on any published image. To use a published +# image instead, replace the syncserver `build:` block with: +# +# image: ghcr.io/mozilla-services/syncstorage-rs/syncstorage-rs-mysql:${SYNCSERVER_VERSION:?set SYNCSERVER_VERSION to a published tag} +# platform: linux/amd64 +# +# Published images are tagged by commit SHA (no `latest`/semver), so +# SYNCSERVER_VERSION must be a tag from the syncstorage-rs-mysql packages page. +# +# Set SYNC_MASTER_SECRET to your own value for anything beyond local +# experimentation; the default below is a placeholder. + +services: + syncserver: + build: + context: .. + args: + SYNCSTORAGE_DATABASE_BACKEND: mysql + TOKENSERVER_DATABASE_BACKEND: mysql + container_name: syncserver + ports: + - "8000:8000" + environment: + SYNC_HOST: "0.0.0.0" + SYNC_PORT: "8000" + SYNC_MASTER_SECRET: "${SYNC_MASTER_SECRET:-changeme_secret_key}" + SYNC_SYNCSTORAGE__DATABASE_URL: "mysql://sync:sync@sync-db:3306/syncstorage" + SYNC_TOKENSERVER__DATABASE_URL: "mysql://sync:sync@tokenserver-db:3306/tokenserver" + SYNC_TOKENSERVER__ENABLED: "true" + SYNC_TOKENSERVER__NODE_TYPE: "mysql" + SYNC_TOKENSERVER__RUN_MIGRATIONS: "true" + SYNC_TOKENSERVER__FXA_EMAIL_DOMAIN: "api.accounts.firefox.com" + SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL: "https://oauth.accounts.firefox.com" + SYNC_HUMAN_LOGS: "${SYNC_HUMAN_LOGS:-false}" + RUST_LOG: "${RUST_LOG:-info}" + SYNC_TOKENSERVER__INIT_NODE_URL: "${SYNC_TOKENSERVER__INIT_NODE_URL:-http://localhost:${SYNC_PORT:-8000}}" + depends_on: + sync-db: + condition: service_healthy + tokenserver-db: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/__heartbeat__"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + sync-db: + image: docker.io/library/mysql:8.0 + container_name: syncserver-sync-db + command: --explicit_defaults_for_timestamp + environment: + MYSQL_RANDOM_ROOT_PASSWORD: "yes" + MYSQL_DATABASE: syncstorage + MYSQL_USER: sync + MYSQL_PASSWORD: sync + volumes: + - sync_db_data:/var/lib/mysql + healthcheck: + test: ["CMD-SHELL", "mysqladmin -h 127.0.0.1 -usync -psync ping"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + + tokenserver-db: + image: docker.io/library/mysql:8.0 + container_name: syncserver-tokenserver-db + command: --explicit_defaults_for_timestamp + environment: + MYSQL_RANDOM_ROOT_PASSWORD: "yes" + MYSQL_DATABASE: tokenserver + MYSQL_USER: sync + MYSQL_PASSWORD: sync + volumes: + - tokenserver_db_data:/var/lib/mysql + healthcheck: + test: ["CMD-SHELL", "mysqladmin -h 127.0.0.1 -usync -psync ping"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + +volumes: + sync_db_data: + driver: local + tokenserver_db_data: + driver: local diff --git a/docker/docker-compose.one-shot.postgres.yaml b/docker/docker-compose.one-shot.postgres.yaml new file mode 100644 index 0000000000..897626fefe --- /dev/null +++ b/docker/docker-compose.one-shot.postgres.yaml @@ -0,0 +1,85 @@ +# One-shot, stand-alone PostgreSQL-backed Syncserver. +# +# Unlike docker-compose.postgres.yaml (which is an end-to-end test harness: +# mock FxA server, prebuilt `app:build` image, separate sync/tokenserver DBs), +# this brings up a working server in a single command with no manual database +# setup: +# +# make docker_oneshot_postgres +# # or: +# docker compose -f docker/docker-compose.one-shot.postgres.yaml up -d --build +# +# Then confirm it is serving: +# +# curl http://localhost:8000/__heartbeat__ +# +# Syncstorage and Tokenserver share a single Postgres database here for +# simplicity; SYNC_TOKENSERVER__RUN_MIGRATIONS applies the Tokenserver schema +# and SYNC_TOKENSERVER__INIT_NODE_URL bootstraps the `sync-1.5` service and +# storage node records. +# +# This builds the Postgres server from the local checkout (build context is the +# repo root), so it does not depend on any published image. To use a published +# image instead, replace the syncserver `build:` block with: +# +# image: ghcr.io/mozilla-services/syncstorage-rs/syncserver-postgres:${SYNCSERVER_VERSION:-latest} +# platform: linux/amd64 +# +# Set SYNC_MASTER_SECRET to your own value for anything beyond local +# experimentation; the default below is a placeholder. + +services: + syncserver: + build: + context: .. + args: + SYNCSTORAGE_DATABASE_BACKEND: postgres + TOKENSERVER_DATABASE_BACKEND: postgres + container_name: syncserver + ports: + - "8000:8000" + environment: + SYNC_HOST: "0.0.0.0" + SYNC_PORT: "8000" + SYNC_MASTER_SECRET: "${SYNC_MASTER_SECRET:-changeme_secret_key}" + SYNC_SYNCSTORAGE__DATABASE_URL: "postgres://sync:sync@postgres:5432/syncserver" + SYNC_TOKENSERVER__DATABASE_URL: "postgres://sync:sync@postgres:5432/syncserver" + SYNC_TOKENSERVER__ENABLED: "true" + SYNC_TOKENSERVER__NODE_TYPE: "postgres" + SYNC_TOKENSERVER__RUN_MIGRATIONS: "true" + SYNC_TOKENSERVER__FXA_EMAIL_DOMAIN: "api.accounts.firefox.com" + SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL: "https://oauth.accounts.firefox.com" + SYNC_HUMAN_LOGS: "${SYNC_HUMAN_LOGS:-false}" + RUST_LOG: "${RUST_LOG:-info}" + SYNC_TOKENSERVER__INIT_NODE_URL: "${SYNC_TOKENSERVER__INIT_NODE_URL:-http://localhost:${SYNC_PORT:-8000}}" + depends_on: + postgres: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/__heartbeat__"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + postgres: + image: docker.io/library/postgres:18 + container_name: syncserver-postgres + environment: + POSTGRES_USER: sync + POSTGRES_PASSWORD: sync + POSTGRES_DB: syncserver + volumes: + - postgres_data:/var/lib/postgresql + healthcheck: + test: ["CMD-SHELL", "pg_isready -U sync -d syncserver"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + +volumes: + postgres_data: + driver: local diff --git a/docker/docker-compose.one-shot.spanner.yaml b/docker/docker-compose.one-shot.spanner.yaml new file mode 100644 index 0000000000..f27591cef5 --- /dev/null +++ b/docker/docker-compose.one-shot.spanner.yaml @@ -0,0 +1,113 @@ +# One-shot, stand-alone Spanner-backed Syncserver (LOCAL DEV ONLY). +# +# This mirrors the production shape — Spanner for Syncstorage, MySQL for +# Tokenserver — but runs against the Cloud Spanner *emulator*, so it needs no +# GCP service account. It is for local experimentation only: the emulator is +# in-memory-ish, single-node, unauthenticated, and not suitable for any real +# data. Production uses real Spanner with a service-account key (see +# `make run_spanner`). +# +# Brings up the stack in a single command with no manual database setup: +# +# make docker_oneshot_spanner +# # or: +# docker compose -f docker/docker-compose.one-shot.spanner.yaml up -d --build +# +# Then confirm it is serving: +# +# curl http://localhost:8000/__heartbeat__ +# +# The sync-db-setup container provisions the emulator instance/database from the +# image's bundled schema (scripts/prepare-spanner.sh + /app/schema.ddl) and then +# exits; syncserver waits for it to finish. Tokenserver applies its schema via +# SYNC_TOKENSERVER__RUN_MIGRATIONS, and SYNC_TOKENSERVER__INIT_NODE_URL +# bootstraps the `sync-1.5` service and storage node records. +# +# The Spanner server is built from the local checkout (build context is the repo +# root); sync-db-setup builds the image and syncserver reuses it. +# +# Set SYNC_MASTER_SECRET to your own value for anything beyond local +# experimentation; the default below is a placeholder. + +services: + sync-db: + # Keep in sync with SPANNER_EMULATOR_VER in .github/workflows/main-workflow.yml + image: gcr.io/cloud-spanner-emulator/emulator:1.5.52 + container_name: syncserver-spanner-emulator + ports: + - "9010:9010" + - "9020:9020" + restart: unless-stopped + + sync-db-setup: + build: + context: .. + args: + SYNCSTORAGE_DATABASE_BACKEND: spanner + TOKENSERVER_DATABASE_BACKEND: mysql + image: syncstorage-rs-spanner-oneshot:latest + container_name: syncserver-spanner-setup + depends_on: + - sync-db + restart: "no" + entrypoint: ["/app/scripts/prepare-spanner.sh"] + environment: + # prepare-spanner.sh talks to the emulator's REST admin port (9020). + SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: sync-db:9020 + + tokenserver-db: + image: docker.io/library/mysql:8.0 + container_name: syncserver-tokenserver-db + command: --explicit_defaults_for_timestamp + environment: + MYSQL_RANDOM_ROOT_PASSWORD: "yes" + MYSQL_DATABASE: tokenserver + MYSQL_USER: sync + MYSQL_PASSWORD: sync + volumes: + - tokenserver_db_data:/var/lib/mysql + healthcheck: + test: ["CMD-SHELL", "mysqladmin -h 127.0.0.1 -usync -psync ping"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + + syncserver: + image: syncstorage-rs-spanner-oneshot:latest + container_name: syncserver + ports: + - "8000:8000" + environment: + SYNC_HOST: "0.0.0.0" + SYNC_PORT: "8000" + SYNC_MASTER_SECRET: "${SYNC_MASTER_SECRET:-changeme_secret_key}" + SYNC_SYNCSTORAGE__DATABASE_URL: "spanner://projects/test-project/instances/test-instance/databases/test-database" + # syncserver uses the emulator's gRPC port (9010). + SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: sync-db:9010 + SYNC_TOKENSERVER__DATABASE_URL: "mysql://sync:sync@tokenserver-db:3306/tokenserver" + SYNC_TOKENSERVER__ENABLED: "true" + SYNC_TOKENSERVER__NODE_TYPE: "spanner" + SYNC_TOKENSERVER__RUN_MIGRATIONS: "true" + SYNC_TOKENSERVER__FXA_EMAIL_DOMAIN: "api.accounts.firefox.com" + SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL: "https://oauth.accounts.firefox.com" + SYNC_HUMAN_LOGS: "${SYNC_HUMAN_LOGS:-false}" + RUST_LOG: "${RUST_LOG:-info}" + SYNC_TOKENSERVER__INIT_NODE_URL: "${SYNC_TOKENSERVER__INIT_NODE_URL:-http://localhost:${SYNC_PORT:-8000}}" + depends_on: + sync-db-setup: + condition: service_completed_successfully + tokenserver-db: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/__heartbeat__"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + +volumes: + tokenserver_db_data: + driver: local diff --git a/docs/src/how-to/how-to-run-with-docker.md b/docs/src/how-to/how-to-run-with-docker.md index 833661d8e6..cbb79f8d92 100644 --- a/docs/src/how-to/how-to-run-with-docker.md +++ b/docs/src/how-to/how-to-run-with-docker.md @@ -76,175 +76,26 @@ SYNC_TOKENSERVER__INIT_NODE_URL="http://localhost:8000" \ docker compose -f docker-compose.yaml up -d ``` -## Docker Compose, One-Shot with PostgreSQL +## Docker Compose, One-Shot Stand-Alone Servers -Alternatively, the database can be started through `docker compose` as well. The real service URL can be set with the `INIT_NODE_URL` environment variable. +The repository ships ready-to-run, stand-alone compose files under `docker/` +that bring up a complete server — database(s) included — in a single command, +with no manual database setup. Each builds the server from your local checkout, +so they work directly from a clone without a published image. From the repo +root: -Save the yaml below into a file, e.g. `docker-compose.one-shot.yaml`. +| Backend | Make target | Compose file | +|---|---|---| +| MySQL | `make docker_oneshot_mysql` | `docker/docker-compose.one-shot.mysql.yaml` | +| PostgreSQL | `make docker_oneshot_postgres` | `docker/docker-compose.one-shot.postgres.yaml` | +| Spanner (emulator, local dev only) | `make docker_oneshot_spanner` | `docker/docker-compose.one-shot.spanner.yaml` | -```yaml -services: - syncserver: - image: ghcr.io/mozilla-services/syncstorage-rs/syncserver-postgres:${SYNCSERVER_VERSION:-latest} - platform: linux/amd64 - container_name: syncserver - ports: - - "8000:8000" - environment: - SYNC_HOST: "0.0.0.0" - SYNC_PORT: "8000" - SYNC_MASTER_SECRET: "${SYNC_MASTER_SECRET:-changeme_secret_key}" - SYNC_SYNCSTORAGE__DATABASE_URL: "postgres://sync:sync@postgres:5432/syncserver" - SYNC_TOKENSERVER__DATABASE_URL: "postgres://sync:sync@postgres:5432/syncserver" - SYNC_TOKENSERVER__ENABLED: "true" - SYNC_TOKENSERVER__RUN_MIGRATIONS: "true" - SYNC_TOKENSERVER__FXA_EMAIL_DOMAIN: "api.accounts.firefox.com" - SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL: "https://oauth.accounts.firefox.com" - SYNC_HUMAN_LOGS: "${SYNC_HUMAN_LOGS:-false}" - RUST_LOG: "${RUST_LOG:-info}" - SYNC_TOKENSERVER__INIT_NODE_URL: "${SYNC_TOKENSERVER__INIT_NODE_URL:-http://localhost:${SYNC_PORT}}" - depends_on: - postgres: - condition: service_healthy - restart: unless-stopped - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:${SYNC_PORT}/__heartbeat__"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 60s - - postgres: - image: postgres:18 - container_name: syncserver-postgres - environment: - POSTGRES_USER: sync - POSTGRES_PASSWORD: sync - POSTGRES_DB: syncserver - volumes: - - postgres_data:/var/lib/postgresql - healthcheck: - test: ["CMD-SHELL", "pg_isready -U sync -d syncserver"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 30s - restart: unless-stopped - -volumes: - postgres_data: - driver: local -``` - -Next, start the service with `docker compose`: - -```sh -docker compose -f docker-compose.one-shot.yaml up -d -``` - -## Docker Compose, One-Shot with MySQL - -This recipe brings up everything needed for a working MySQL-backed server in a -single command: a MySQL database for Syncstorage and one for Tokenserver, plus -the server itself. Syncstorage applies its schema migrations automatically at -startup, `SYNC_TOKENSERVER__RUN_MIGRATIONS` applies the Tokenserver schema, and -`SYNC_TOKENSERVER__INIT_NODE_URL` bootstraps the `sync-1.5` service and storage -node records — so the stack is ready to serve with no manual database setup. - -### Option A: build from source (works from a checkout) - -Run this from a clone of the repository; the build `context` is the repo root, -so the MySQL build of the server is compiled locally and the recipe does not -depend on any published image. Save the yaml below into a file, e.g. -`docker-compose.one-shot.yaml`. - -```yaml -services: - syncserver: - build: - context: . - args: - SYNCSTORAGE_DATABASE_BACKEND: mysql - TOKENSERVER_DATABASE_BACKEND: mysql - container_name: syncserver - ports: - - "8000:8000" - environment: - SYNC_HOST: "0.0.0.0" - SYNC_PORT: "8000" - SYNC_MASTER_SECRET: "${SYNC_MASTER_SECRET:-changeme_secret_key}" - SYNC_SYNCSTORAGE__DATABASE_URL: "mysql://sync:sync@sync-db:3306/syncstorage" - SYNC_TOKENSERVER__DATABASE_URL: "mysql://sync:sync@tokenserver-db:3306/tokenserver" - SYNC_TOKENSERVER__ENABLED: "true" - SYNC_TOKENSERVER__NODE_TYPE: "mysql" - SYNC_TOKENSERVER__RUN_MIGRATIONS: "true" - SYNC_TOKENSERVER__FXA_EMAIL_DOMAIN: "api.accounts.firefox.com" - SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL: "https://oauth.accounts.firefox.com" - SYNC_HUMAN_LOGS: "${SYNC_HUMAN_LOGS:-false}" - RUST_LOG: "${RUST_LOG:-info}" - SYNC_TOKENSERVER__INIT_NODE_URL: "${SYNC_TOKENSERVER__INIT_NODE_URL:-http://localhost:${SYNC_PORT:-8000}}" - depends_on: - sync-db: - condition: service_healthy - tokenserver-db: - condition: service_healthy - restart: unless-stopped - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/__heartbeat__"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 60s - - sync-db: - image: docker.io/library/mysql:8.0 - container_name: syncserver-sync-db - command: --explicit_defaults_for_timestamp - environment: - MYSQL_RANDOM_ROOT_PASSWORD: "yes" - MYSQL_DATABASE: syncstorage - MYSQL_USER: sync - MYSQL_PASSWORD: sync - volumes: - - sync_db_data:/var/lib/mysql - healthcheck: - test: ["CMD-SHELL", "mysqladmin -h 127.0.0.1 -usync -psync ping"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 30s - restart: unless-stopped - - tokenserver-db: - image: docker.io/library/mysql:8.0 - container_name: syncserver-tokenserver-db - command: --explicit_defaults_for_timestamp - environment: - MYSQL_RANDOM_ROOT_PASSWORD: "yes" - MYSQL_DATABASE: tokenserver - MYSQL_USER: sync - MYSQL_PASSWORD: sync - volumes: - - tokenserver_db_data:/var/lib/mysql - healthcheck: - test: ["CMD-SHELL", "mysqladmin -h 127.0.0.1 -usync -psync ping"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 30s - restart: unless-stopped - -volumes: - sync_db_data: - driver: local - tokenserver_db_data: - driver: local -``` - -Next, build and start the service with `docker compose`: +For example, for MySQL: ```sh -docker compose -f docker-compose.one-shot.yaml up -d --build +make docker_oneshot_mysql +# equivalently: +docker compose -f docker/docker-compose.one-shot.mysql.yaml up -d --build ``` Once the `syncserver` container reports healthy, confirm it is serving: @@ -253,34 +104,51 @@ Once the `syncserver` container reports healthy, confirm it is serving: curl http://localhost:8000/__heartbeat__ ``` -### Option B: use a published image +Stop and remove a stack with the matching `_stop` target, e.g. +`make docker_oneshot_mysql_stop`. -Mozilla also publishes prebuilt images on ghcr.io. Note that these are -currently tagged by commit SHA — there is **no `latest` or semver tag** — so -you must pin `SYNCSERVER_VERSION` to a tag listed on the +Syncstorage applies its schema migrations at startup, +`SYNC_TOKENSERVER__RUN_MIGRATIONS` applies the Tokenserver schema, and +`SYNC_TOKENSERVER__INIT_NODE_URL` bootstraps the `sync-1.5` service and storage +node records — so the stack is ready to serve immediately. + +> Set `SYNC_MASTER_SECRET` to your own value for anything beyond local +> experimentation; the compose files default to a placeholder. + +### Backend notes + +- **MySQL / PostgreSQL** are reasonable starting points for a real self-hosted + deployment. The MySQL recipe uses a separate database for Syncstorage and + Tokenserver; the PostgreSQL recipe shares a single database between them for + simplicity. +- **Spanner** runs against the Cloud Spanner *emulator* and is for local + experimentation only — it is unauthenticated, single-node, and not durable. + Production Spanner uses a real instance and a service-account key (see + `make run_spanner`). The recipe mirrors the production split (Spanner for + Syncstorage, MySQL for Tokenserver) and includes a one-time setup container + that provisions the emulator's schema before the server starts. + +### Using a published image instead of building + +Mozilla also publishes prebuilt images on ghcr.io. The MySQL images are +currently tagged by commit SHA — there is **no `latest` or semver tag** — so you +must pin `SYNCSERVER_VERSION` to a tag listed on the [`syncstorage-rs-mysql` packages page](https://github.com/mozilla-services/syncstorage-rs/pkgs/container/syncstorage-rs%2Fsyncstorage-rs-mysql). -To use a published image, replace the `syncserver` service's `build:` block with -an `image:` reference; the `sync-db`, `tokenserver-db`, and `volumes` sections -are unchanged: +To use one, replace the `syncserver` service's `build:` block in the compose +file with an `image:` reference (published images are `linux/amd64`): ```yaml services: syncserver: image: ghcr.io/mozilla-services/syncstorage-rs/syncstorage-rs-mysql:${SYNCSERVER_VERSION:?set SYNCSERVER_VERSION to a published tag} platform: linux/amd64 - container_name: syncserver - # ...the remaining syncserver settings are identical to Option A + # ...the remaining syncserver settings are unchanged ``` -Then start it with the tag pinned (published images are `linux/amd64`): - ```sh -SYNCSERVER_VERSION= docker compose -f docker-compose.one-shot.yaml up -d +SYNCSERVER_VERSION= docker compose -f docker/docker-compose.one-shot.mysql.yaml up -d ``` -> Set `SYNC_MASTER_SECRET` to your own value for anything beyond local -> experimentation; the default above is a placeholder. - ## Configuring Firefox (Desktop) Firefox itself needs to be configured to use the self-hosted Sync server. From a6be314ae2d2f1265cecdba388987ea339d700c4 Mon Sep 17 00:00:00 2001 From: Taddes Date: Wed, 24 Jun 2026 17:06:30 -0400 Subject: [PATCH 7/7] fix(docker): parameterize SYNC_PORT --- docker/docker-compose.one-shot.mysql.yaml | 6 +++--- docker/docker-compose.one-shot.postgres.yaml | 6 +++--- docker/docker-compose.one-shot.spanner.yaml | 6 +++--- docs/src/how-to/how-to-run-with-docker.md | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docker/docker-compose.one-shot.mysql.yaml b/docker/docker-compose.one-shot.mysql.yaml index 5f6320b122..7a607d4949 100644 --- a/docker/docker-compose.one-shot.mysql.yaml +++ b/docker/docker-compose.one-shot.mysql.yaml @@ -39,10 +39,10 @@ services: TOKENSERVER_DATABASE_BACKEND: mysql container_name: syncserver ports: - - "8000:8000" + - "${SYNC_PORT:-8000}:${SYNC_PORT:-8000}" environment: SYNC_HOST: "0.0.0.0" - SYNC_PORT: "8000" + SYNC_PORT: "${SYNC_PORT:-8000}" SYNC_MASTER_SECRET: "${SYNC_MASTER_SECRET:-changeme_secret_key}" SYNC_SYNCSTORAGE__DATABASE_URL: "mysql://sync:sync@sync-db:3306/syncstorage" SYNC_TOKENSERVER__DATABASE_URL: "mysql://sync:sync@tokenserver-db:3306/tokenserver" @@ -61,7 +61,7 @@ services: condition: service_healthy restart: unless-stopped healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/__heartbeat__"] + test: ["CMD", "curl", "-f", "http://localhost:${SYNC_PORT:-8000}/__heartbeat__"] interval: 30s timeout: 10s retries: 3 diff --git a/docker/docker-compose.one-shot.postgres.yaml b/docker/docker-compose.one-shot.postgres.yaml index 897626fefe..9823360e65 100644 --- a/docker/docker-compose.one-shot.postgres.yaml +++ b/docker/docker-compose.one-shot.postgres.yaml @@ -37,10 +37,10 @@ services: TOKENSERVER_DATABASE_BACKEND: postgres container_name: syncserver ports: - - "8000:8000" + - "${SYNC_PORT:-8000}:${SYNC_PORT:-8000}" environment: SYNC_HOST: "0.0.0.0" - SYNC_PORT: "8000" + SYNC_PORT: "${SYNC_PORT:-8000}" SYNC_MASTER_SECRET: "${SYNC_MASTER_SECRET:-changeme_secret_key}" SYNC_SYNCSTORAGE__DATABASE_URL: "postgres://sync:sync@postgres:5432/syncserver" SYNC_TOKENSERVER__DATABASE_URL: "postgres://sync:sync@postgres:5432/syncserver" @@ -57,7 +57,7 @@ services: condition: service_healthy restart: unless-stopped healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/__heartbeat__"] + test: ["CMD", "curl", "-f", "http://localhost:${SYNC_PORT:-8000}/__heartbeat__"] interval: 30s timeout: 10s retries: 3 diff --git a/docker/docker-compose.one-shot.spanner.yaml b/docker/docker-compose.one-shot.spanner.yaml index f27591cef5..f31ad2b0e5 100644 --- a/docker/docker-compose.one-shot.spanner.yaml +++ b/docker/docker-compose.one-shot.spanner.yaml @@ -78,10 +78,10 @@ services: image: syncstorage-rs-spanner-oneshot:latest container_name: syncserver ports: - - "8000:8000" + - "${SYNC_PORT:-8000}:${SYNC_PORT:-8000}" environment: SYNC_HOST: "0.0.0.0" - SYNC_PORT: "8000" + SYNC_PORT: "${SYNC_PORT:-8000}" SYNC_MASTER_SECRET: "${SYNC_MASTER_SECRET:-changeme_secret_key}" SYNC_SYNCSTORAGE__DATABASE_URL: "spanner://projects/test-project/instances/test-instance/databases/test-database" # syncserver uses the emulator's gRPC port (9010). @@ -102,7 +102,7 @@ services: condition: service_healthy restart: unless-stopped healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/__heartbeat__"] + test: ["CMD", "curl", "-f", "http://localhost:${SYNC_PORT:-8000}/__heartbeat__"] interval: 30s timeout: 10s retries: 3 diff --git a/docs/src/how-to/how-to-run-with-docker.md b/docs/src/how-to/how-to-run-with-docker.md index cbb79f8d92..a3544dd090 100644 --- a/docs/src/how-to/how-to-run-with-docker.md +++ b/docs/src/how-to/how-to-run-with-docker.md @@ -37,10 +37,10 @@ services: platform: linux/amd64 container_name: syncserver ports: - - "8000:8000" + - "${SYNC_PORT:-8000}:${SYNC_PORT:-8000}" environment: SYNC_HOST: "0.0.0.0" - SYNC_PORT: "8000" + SYNC_PORT: "${SYNC_PORT:-8000}" SYNC_MASTER_SECRET: "${SYNC_MASTER_SECRET}" SYNC_SYNCSTORAGE__DATABASE_URL: "${SYNC_SYNCSTORAGE__DATABASE_URL}" SYNC_TOKENSERVER__DATABASE_URL: "${SYNC_TOKENSERVER__DATABASE_URL}" @@ -48,10 +48,10 @@ services: SYNC_TOKENSERVER__RUN_MIGRATIONS: "true" SYNC_TOKENSERVER__FXA_EMAIL_DOMAIN: "api.accounts.firefox.com" SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL: "https://oauth.accounts.firefox.com" - SYNC_TOKENSERVER__INIT_NODE_URL: "${SYNC_TOKENSERVER__INIT_NODE_URL:-http://localhost:${SYNC_PORT}}" + SYNC_TOKENSERVER__INIT_NODE_URL: "${SYNC_TOKENSERVER__INIT_NODE_URL:-http://localhost:${SYNC_PORT:-8000}}" restart: unless-stopped healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:${SYNC_PORT}/__heartbeat__"] + test: ["CMD", "curl", "-f", "http://localhost:${SYNC_PORT:-8000}/__heartbeat__"] interval: 30s timeout: 10s retries: 3