From deb512f9f53404d6af0593fd0ad6b9e6d210e06b Mon Sep 17 00:00:00 2001 From: Daniel Gerlag Date: Thu, 21 May 2026 16:37:51 -0700 Subject: [PATCH 1/3] mysql Signed-off-by: Daniel Gerlag --- .../configure-bootstrap-providers/_index.md | 9 + .../_index.md | 107 ++++++++ .../configuration/configure-sources/_index.md | 9 + .../configure-mysql-source/_index.md | 253 ++++++++++++++++++ 4 files changed, 378 insertions(+) create mode 100644 docs/content/drasi-server/how-to-guides/configuration/configure-bootstrap-providers/configure-mysql-bootstrap-provider/_index.md create mode 100644 docs/content/drasi-server/how-to-guides/configuration/configure-sources/configure-mysql-source/_index.md diff --git a/docs/content/drasi-server/how-to-guides/configuration/configure-bootstrap-providers/_index.md b/docs/content/drasi-server/how-to-guides/configuration/configure-bootstrap-providers/_index.md index 662e3cb4..305746b1 100644 --- a/docs/content/drasi-server/how-to-guides/configuration/configure-bootstrap-providers/_index.md +++ b/docs/content/drasi-server/how-to-guides/configuration/configure-bootstrap-providers/_index.md @@ -32,6 +32,15 @@ description: "Load initial data before streaming begins" --> + +
+
+
+

MySQL

+

Bootstrap from a MySQL snapshot (MySQL sources only)

+
+
+
diff --git a/docs/content/drasi-server/how-to-guides/configuration/configure-bootstrap-providers/configure-mysql-bootstrap-provider/_index.md b/docs/content/drasi-server/how-to-guides/configuration/configure-bootstrap-providers/configure-mysql-bootstrap-provider/_index.md new file mode 100644 index 00000000..809b3899 --- /dev/null +++ b/docs/content/drasi-server/how-to-guides/configuration/configure-bootstrap-providers/configure-mysql-bootstrap-provider/_index.md @@ -0,0 +1,107 @@ +--- +type: "docs" +title: "Configure MySQL Bootstrap Provider" +linkTitle: "MySQL" +weight: 45 +description: "Bootstrap queries from a MySQL snapshot" +related: + howto: + - title: "Configure Sources" + url: "/drasi-server/how-to-guides/configuration/configure-sources/" + - title: "Configure MySQL Source" + url: "/drasi-server/how-to-guides/configuration/configure-sources/configure-mysql-source/" + - title: "Configure Bootstrap Providers" + url: "/drasi-server/how-to-guides/configuration/configure-bootstrap-providers/" +--- + +The **MySQL bootstrap provider** loads initial state from a MySQL database so queries start with a complete snapshot before streaming begins. + +## When to use MySQL bootstrap + +- You need historical/current state from MySQL when a query starts. +- Your query depends on existing rows (aggregations, joins, thresholds). +- You want snapshot + binlog streaming from the same database. + +## Prerequisites + +- Use with a **MySQL source** (`sources[].kind: mysql`). +- The database user must have **SELECT** permission on the configured tables. +- The `tables` list is **required** — you must explicitly specify which tables to bootstrap. + +## Quick example (Drasi Server config) + +In Drasi Server config, bootstrap provider keys are **camelCase**, and the discriminator field is `bootstrapProvider.kind`. + +```yaml +sources: + - kind: mysql + id: orders-db + autoStart: true + + host: ${MYSQL_HOST:-localhost} + port: ${MYSQL_PORT:-3306} + database: ${MYSQL_DATABASE:-mydb} + user: ${MYSQL_USER:-drasi_user} + password: ${MYSQL_PASSWORD} + + tables: + - orders + - customers + + bootstrapProvider: + kind: mysql +``` + +## Configuration reference + +| Field | Type | Required | Description | +|---|---|---:|---| +| `kind` | string | Yes | Must be `mysql`. | +| `host` | string | No | MySQL host (default: `localhost`). | +| `port` | integer | No | MySQL port (default: `3306`). | +| `database` | string | Yes | Database name to connect to. | +| `user` | string | Yes | Database user with SELECT permission. | +| `password` | string | No | Password (default: `""`). | +| `tables` | string[] | Yes | Table allow-list for bootstrapping. Must contain at least one table. | +| `tableKeys` | array | No | Override key columns per table (see below). | + +### tableKeys + +Use `tableKeys` to define key columns for element ID generation when primary keys are missing. + +```yaml +bootstrapProvider: + kind: mysql + tableKeys: + - table: order_items + keyColumns: [order_id, product_id] +``` + +## Notes + +- Drasi Server only allows `kind: mysql` when the source is also `kind: mysql`. +- The `tables` list acts as a security allow-list — only tables explicitly listed will be bootstrapped. +- Table names must use only letters, numbers, and underscores. + +## Documentation resources + +
+ +
+
+
+

MySQL Bootstrap README

+

Implementation notes and behavior details

+
+
+ + +
+
+
+

drasi-bootstrap-mysql on crates.io

+

Package info and release history

+
+
+ +
diff --git a/docs/content/drasi-server/how-to-guides/configuration/configure-sources/_index.md b/docs/content/drasi-server/how-to-guides/configuration/configure-sources/_index.md index 43c288f7..374c17d4 100644 --- a/docs/content/drasi-server/how-to-guides/configuration/configure-sources/_index.md +++ b/docs/content/drasi-server/how-to-guides/configuration/configure-sources/_index.md @@ -48,6 +48,15 @@ description: "Connect Drasi Server to databases, APIs, and data streams"
--> + +
+
+
+

MySQL

+

Stream changes from MySQL using binlog replication

+
+
+
diff --git a/docs/content/drasi-server/how-to-guides/configuration/configure-sources/configure-mysql-source/_index.md b/docs/content/drasi-server/how-to-guides/configuration/configure-sources/configure-mysql-source/_index.md new file mode 100644 index 00000000..39944fc4 --- /dev/null +++ b/docs/content/drasi-server/how-to-guides/configuration/configure-sources/configure-mysql-source/_index.md @@ -0,0 +1,253 @@ +--- +type: "docs" +title: "Configure MySQL Source" +linkTitle: "MySQL" +weight: 55 +description: "Stream changes from MySQL using binlog replication" +related: + concepts: + - title: "Sources" + url: "/concepts/sources/" + howto: + - title: "Configure Bootstrap Providers" + url: "/drasi-server/how-to-guides/configuration/configure-bootstrap-providers/" + reference: + - title: "Configuration Reference" + url: "/drasi-server/reference/configuration/" +--- + +The MySQL {{< term "Source" >}} streams row-level changes from a MySQL database using **binary log (binlog) replication**. + +## When to use the MySQL source + +- Keep Drasi queries continuously updated from a system-of-record MySQL database. +- Drive reactions from database changes (alerts, notifications, downstream sync, cache/materialized-view updates). +- Build reactive services that need transactional ordering of changes. + +## Prerequisites + +- MySQL **5.7+** or **8.0+**. +- Binary logging enabled with row-based format: + - `binlog_format = ROW` + - `binlog_row_image = FULL` + - `binlog_row_metadata = FULL` +- A database user with **REPLICATION SLAVE**, **REPLICATION CLIENT**, and **SELECT** permissions. + +## How it connects + +This source **connects outbound** from Drasi Server to MySQL over the MySQL protocol; it does not open an inbound port. + +## Quick example (Drasi Server config) + +Drasi Server source configuration uses **camelCase** keys. + +```yaml +sources: + - kind: mysql + id: orders-db + autoStart: true + + host: ${MYSQL_HOST:-localhost} + port: ${MYSQL_PORT:-3306} + database: ${MYSQL_DATABASE:-mydb} + user: ${MYSQL_USER:-drasi_user} + password: ${MYSQL_PASSWORD} + + # Tables to monitor + tables: + - orders + - customers + + # Optional: override key columns for tables without primary keys + tableKeys: + - table: order_items + keyColumns: [order_id, product_id] + + # Optional: where to start reading the binlog + startPosition: from_end + + # Optional: preload initial state for newly-subscribed queries + bootstrapProvider: + kind: mysql +``` + +## Configure MySQL + +### 1) Enable binary logging with row format + +Add to your MySQL configuration (`my.cnf` or `my.ini`) and restart MySQL: + +```ini +[mysqld] +log-bin = mysql-bin +binlog_format = ROW +binlog_row_image = FULL +binlog_row_metadata = FULL +server-id = 1 +``` + +### 2) Create a replication user + +```sql +CREATE USER 'drasi_user'@'%' IDENTIFIED BY 'your-password'; +GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'drasi_user'@'%'; +GRANT SELECT ON mydb.* TO 'drasi_user'@'%'; +FLUSH PRIVILEGES; +``` + +### 3) (Recommended) Ensure tables have primary keys + +For reliable change tracking, tables should have primary keys. +If replicating tables without primary keys, configure `tableKeys` (below). + +## Data mapping + +- Each changed row becomes a Drasi graph {{< term "Node" >}}. +- **Label**: table name (for example `orders`). +- **Properties**: columns become node properties. +- **Element ID**: `table:key` (for example `orders:123`). + +If no key columns can be resolved for a row, the source logs a warning and falls back to a generated UUID: `table:uuid`. + +## Configuration reference (Drasi Server) + +| Field | Type | Default | Description | +|---|---:|---:|---| +| `kind` | string | required | Must be `mysql`. | +| `id` | string | required | Unique source identifier. | +| `autoStart` | boolean | `true` | Whether Drasi Server starts the source on startup. | +| `bootstrapProvider` | object | none | Optional bootstrap provider for initial state. For MySQL bootstrap, use `{ kind: mysql }`. | +| `host` | string | `localhost` | MySQL host. | +| `port` | integer | `3306` | MySQL port. | +| `database` | string | required | Database name. | +| `user` | string | required | Database user (must have replication permission). | +| `password` | string | `""` | Password. | +| `tables` | string[] | `[]` | List of tables to monitor for changes. | +| `sslMode` | string | `disabled` | SSL mode: `disabled`, `if_available`, `require`, `require_verify_ca`, `require_verify_full`. | +| `tableKeys` | array | `[]` | Override key columns per table (see below). | +| `startPosition` | string | `from_end` | Where to start the binlog stream: `from_start`, `from_end`, `from_position`, `from_gtid`. | +| `serverId` | integer | auto-generated | MySQL server ID for the replication connection. Auto-generated from source instance ID if not specified. | +| `heartbeatIntervalSeconds` | integer | `30` | Heartbeat interval in seconds for the replication connection. | + +Fields marked with support Drasi Server config references like `${ENV_VAR}` / `${ENV_VAR:-default}`. + +### tableKeys + +Use `tableKeys` to define key columns for element ID generation when primary keys are missing or not suitable. + +```yaml +tableKeys: + - table: order_items + keyColumns: [order_id, product_id] +``` + +Notes: +- Key columns are joined with `_` in the element ID (for example `order_items:1001_5`). + +### startPosition + +Controls where binlog replication begins when the source starts for the first time. + +| Value | Description | +|---|---| +| `from_end` | Start from the current end of the binlog (default). Only captures new changes. | +| `from_start` | Replay the binlog from the beginning. | +| `from_position` | Start from a specific binlog file and position. | +| `from_gtid` | Start from a specific GTID set. | + +## Performance tuning notes + +- Only list the tables you need in `tables`; this reduces processing overhead. +- The `heartbeatIntervalSeconds` prevents idle connection timeouts and allows the source to detect disconnections faster. +- Monitor binlog retention to ensure the source can reconnect after brief outages. + +## Verifying it works + +After starting Drasi Server with your MySQL source, verify the connection: + +### 1. Check source status + +```bash +curl http://localhost:8080/api/v1/sources/orders-db +``` + +Expected response includes `"status": "running"`. + +### 2. Make a test change in MySQL + +```sql +INSERT INTO orders (id, customer_id, total, status) +VALUES (999, 1, 100.00, 'pending'); +``` + +### 3. Verify the change was captured + +If you have a log reaction configured: + +``` +[console-output] Query 'my-query' (1 items): +[console-output] [ADD] {"id":"999","customer_id":"1","total":"100.00","status":"pending"} +``` + +Or query results via API: + +```bash +curl http://localhost:8080/api/v1/queries/my-query/results +``` + +### 4. Check binlog status in MySQL + +```sql +SHOW BINARY LOG STATUS; +SHOW REPLICAS; +``` + +## Troubleshooting + +| Error | Cause | Solution | +|-------|-------|----------| +| `binlog_format must be ROW` | Binlog format not set to ROW | Set `binlog_format = ROW` in `my.cnf` and restart | +| `Access denied; you need the REPLICATION SLAVE privilege` | Missing permission | `GRANT REPLICATION SLAVE ON *.* TO 'drasi_user'@'%';` | +| `Can't connect to MySQL server` | Wrong host/port | Verify `host` and `port` values, check firewall | +| No changes captured | Table not in `tables` list | Add the table to the `tables` configuration | +| Unstable element IDs | No primary key | Add primary key or configure `tableKeys` | +| `binlog_row_image must be FULL` | Row image not complete | Set `binlog_row_image = FULL` in `my.cnf` and restart | + +### MySQL Permissions Checklist + +Your database user needs: + +```sql +-- Required permissions +GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'drasi_user'@'%'; +GRANT SELECT ON mydb.* TO 'drasi_user'@'%'; +FLUSH PRIVILEGES; +``` + +## Known limitations + +- Packets larger than 16 MB are not supported. +- DDL changes (schema migrations) during streaming may require a source restart. + +## Documentation resources + +
+ +
+
+
+

MySQL Source README

+

Implementation notes, prerequisites, and behavior details

+
+
+ + +
+
+
+

drasi-source-mysql on crates.io

+

Package info and release history

+
+
+ +
From 894a5512ca747560872bc309ce8075c71bfb80fd Mon Sep 17 00:00:00 2001 From: Daniel Gerlag Date: Thu, 21 May 2026 16:43:08 -0700 Subject: [PATCH 2/3] spelling Signed-off-by: Daniel Gerlag --- .github/config/en-drasi.txt | 1 + .../configure-sources/configure-mysql-source/_index.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/config/en-drasi.txt b/.github/config/en-drasi.txt index 7b7bd12b..449c9cc2 100644 --- a/.github/config/en-drasi.txt +++ b/.github/config/en-drasi.txt @@ -210,6 +210,7 @@ SignalRHub slidingWindow sortBy SourceConfirm +startPosition sourceprovider SourceProvider SourceProviders diff --git a/docs/content/drasi-server/how-to-guides/configuration/configure-sources/configure-mysql-source/_index.md b/docs/content/drasi-server/how-to-guides/configuration/configure-sources/configure-mysql-source/_index.md index 39944fc4..75c73219 100644 --- a/docs/content/drasi-server/how-to-guides/configuration/configure-sources/configure-mysql-source/_index.md +++ b/docs/content/drasi-server/how-to-guides/configuration/configure-sources/configure-mysql-source/_index.md @@ -227,7 +227,7 @@ FLUSH PRIVILEGES; ## Known limitations - Packets larger than 16 MB are not supported. -- DDL changes (schema migrations) during streaming may require a source restart. +- Schema changes (migrations, column additions/removals) during streaming may require a source restart. ## Documentation resources From 0af0ecccd358293de3a8b6d51d6aaa73dd50d50a Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 21 May 2026 18:32:00 -0700 Subject: [PATCH 3/3] Resolve remaining MySQL docs review thread inconsistencies (#218) * Initial plan * Fix MySQL docs review feedback in PR #217 --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> --- .../_index.md | 24 ++++--------------- .../configure-mysql-source/_index.md | 7 +++--- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/docs/content/drasi-server/how-to-guides/configuration/configure-bootstrap-providers/configure-mysql-bootstrap-provider/_index.md b/docs/content/drasi-server/how-to-guides/configuration/configure-bootstrap-providers/configure-mysql-bootstrap-provider/_index.md index 809b3899..de9d4fef 100644 --- a/docs/content/drasi-server/how-to-guides/configuration/configure-bootstrap-providers/configure-mysql-bootstrap-provider/_index.md +++ b/docs/content/drasi-server/how-to-guides/configuration/configure-bootstrap-providers/configure-mysql-bootstrap-provider/_index.md @@ -26,7 +26,7 @@ The **MySQL bootstrap provider** loads initial state from a MySQL database so qu - Use with a **MySQL source** (`sources[].kind: mysql`). - The database user must have **SELECT** permission on the configured tables. -- The `tables` list is **required** — you must explicitly specify which tables to bootstrap. +- Configure `tables` (and optional `tableKeys`) on the MySQL source. ## Quick example (Drasi Server config) @@ -54,32 +54,16 @@ sources: ## Configuration reference +`mysql` accepts **no additional fields**. + | Field | Type | Required | Description | |---|---|---:|---| | `kind` | string | Yes | Must be `mysql`. | -| `host` | string | No | MySQL host (default: `localhost`). | -| `port` | integer | No | MySQL port (default: `3306`). | -| `database` | string | Yes | Database name to connect to. | -| `user` | string | Yes | Database user with SELECT permission. | -| `password` | string | No | Password (default: `""`). | -| `tables` | string[] | Yes | Table allow-list for bootstrapping. Must contain at least one table. | -| `tableKeys` | array | No | Override key columns per table (see below). | - -### tableKeys - -Use `tableKeys` to define key columns for element ID generation when primary keys are missing. - -```yaml -bootstrapProvider: - kind: mysql - tableKeys: - - table: order_items - keyColumns: [order_id, product_id] -``` ## Notes - Drasi Server only allows `kind: mysql` when the source is also `kind: mysql`. +- Connection and table scope come from the MySQL source configuration (for example `host`, `database`, `tables`, and `tableKeys`). - The `tables` list acts as a security allow-list — only tables explicitly listed will be bootstrapped. - Table names must use only letters, numbers, and underscores. diff --git a/docs/content/drasi-server/how-to-guides/configuration/configure-sources/configure-mysql-source/_index.md b/docs/content/drasi-server/how-to-guides/configuration/configure-sources/configure-mysql-source/_index.md index 75c73219..581b1262 100644 --- a/docs/content/drasi-server/how-to-guides/configuration/configure-sources/configure-mysql-source/_index.md +++ b/docs/content/drasi-server/how-to-guides/configuration/configure-sources/configure-mysql-source/_index.md @@ -26,7 +26,7 @@ The MySQL {{< term "Source" >}} streams row-level changes from a MySQL database ## Prerequisites -- MySQL **5.7+** or **8.0+**. +- MySQL **8.0+**. - Binary logging enabled with row-based format: - `binlog_format = ROW` - `binlog_row_image = FULL` @@ -129,7 +129,7 @@ If no key columns can be resolved for a row, the source logs a warning and falls | `serverId` | integer | auto-generated | MySQL server ID for the replication connection. Auto-generated from source instance ID if not specified. | | `heartbeatIntervalSeconds` | integer | `30` | Heartbeat interval in seconds for the replication connection. | -Fields marked with support Drasi Server config references like `${ENV_VAR}` / `${ENV_VAR:-default}`. +Fields support Drasi Server config references like `${ENV_VAR}` / `${ENV_VAR:-default}`. ### tableKeys @@ -198,8 +198,7 @@ curl http://localhost:8080/api/v1/queries/my-query/results ### 4. Check binlog status in MySQL ```sql -SHOW BINARY LOG STATUS; -SHOW REPLICAS; +SHOW MASTER STATUS; ``` ## Troubleshooting