Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/rename-durable-streams-rust.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
'@electric-ax/durable-streams-server-rust': patch
---

Rename the package folder to `packages/durable-streams-rust`. Drop the Intel
macOS (`macos-13`) build from the release matrix. Correct the README throughput
figure.
2 changes: 1 addition & 1 deletion .github/workflows/changesets_release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ jobs:
# across unrelated releases that leave the server-rust version unchanged),
# and correct on the first run. The crate version is synced to this anchor
# at publish time by sync-cargo-version.mjs.
VERSION="$(node -p "require('./packages/server-rust/package.json').version")"
VERSION="$(node -p "require('./packages/durable-streams-rust/package.json').version")"
if curl -sf -H 'User-Agent: electric-sql-release-ci' "https://crates.io/api/v1/crates/durable-streams/$VERSION" >/dev/null 2>&1; then
echo "needs_release=false" >> "$GITHUB_OUTPUT"
else
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/server_rust_dockerhub_image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ on:
# push:
# branches: ['main']
# paths:
# - 'packages/server-rust/**'
# - 'packages/durable-streams-rust/**'
# Called by the Changesets release workflow after a version bump.
workflow_call:
inputs:
Expand Down Expand Up @@ -76,7 +76,7 @@ jobs:
run: |
# The crate version lives in the package.json anchor (changeset-managed),
# which is private — so there is no git tag to describe; read it directly.
VERSION=$(node -p "require('./packages/server-rust/package.json').version")
VERSION=$(node -p "require('./packages/durable-streams-rust/package.json').version")

if [ -n "$INPUT_DOCKER_TAG" ]; then
IMAGE_TAGS="${DOCKERHUB_REPO}:${INPUT_DOCKER_TAG}"
Expand Down Expand Up @@ -105,5 +105,5 @@ jobs:
image_repo: electricax/durable-streams-server-rust
artifact_prefix: durable-streams-server-rust
context: .
file: packages/server-rust/Dockerfile
file: packages/durable-streams-rust/Dockerfile
tags: ${{ needs.derive_build_vars.outputs.image_tags }}
20 changes: 11 additions & 9 deletions .github/workflows/server_rust_publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ name: Publish durable-streams-server-rust (npm + crates.io)
# Publishes the binary npm packages (main + 4 platform) and the crate. The Docker
# image is published separately by server_rust_dockerhub_image.yml.
#
# The version is read from packages/server-rust/package.json (the changeset-managed
# The version is read from packages/durable-streams-rust/package.json (the changeset-managed
# anchor, `private: true` so changesets bumps it but does not publish it). Both
# registries use OIDC Trusted Publishing — no tokens (one-time setup required on
# crates.io and npm before the first release).
Expand Down Expand Up @@ -42,11 +42,13 @@ jobs:
target: aarch64-unknown-linux-gnu,
os: blacksmith-4vcpu-ubuntu-2404-arm,
}
- { target: x86_64-apple-darwin, os: macos-13 }
# x86_64-apple-darwin (macos-13, Intel) is disabled: the scarce Intel
# macOS runner pool stalls the release, and its binary only feeds the
# currently-disabled npm packages. Re-add alongside re-enabling npm.
- { target: aarch64-apple-darwin, os: macos-latest }
defaults:
run:
working-directory: packages/server-rust
working-directory: packages/durable-streams-rust
steps:
- uses: actions/checkout@v4
with:
Expand All @@ -58,7 +60,7 @@ jobs:

- uses: Swatinem/rust-cache@v2
with:
workspaces: packages/server-rust
workspaces: packages/durable-streams-rust

- name: Build
run: cargo build --release --locked --target ${{ matrix.target }}
Expand All @@ -81,7 +83,7 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: bin-${{ matrix.target }}
path: packages/server-rust/target/${{ matrix.target }}/release/durable-streams-server
path: packages/durable-streams-rust/target/${{ matrix.target }}/release/durable-streams-server
if-no-files-found: error

cargo-publish:
Expand All @@ -93,15 +95,15 @@ jobs:
ref: ${{ inputs.git_ref }}
- uses: dtolnay/rust-toolchain@stable
- name: Sync Cargo.toml version from the package.json anchor
run: node packages/server-rust/scripts/sync-cargo-version.mjs
run: node packages/durable-streams-rust/scripts/sync-cargo-version.mjs
- name: Authenticate to crates.io (Trusted Publishing, OIDC)
uses: rust-lang/crates-io-auth-action@v1
id: auth
- name: cargo publish
env:
CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
# --allow-dirty: sync-cargo-version edited Cargo.toml in the working tree.
run: cargo publish --allow-dirty --manifest-path packages/server-rust/Cargo.toml
run: cargo publish --allow-dirty --manifest-path packages/durable-streams-rust/Cargo.toml

npm-publish:
needs: build
Expand Down Expand Up @@ -135,8 +137,8 @@ jobs:
done
- name: Assemble npm packages
run: |
version="$(node -p "require('./packages/server-rust/package.json').version")"
node packages/server-rust/npm/assemble.mjs \
version="$(node -p "require('./packages/durable-streams-rust/package.json').version")"
node packages/durable-streams-rust/npm/assemble.mjs \
--version "$version" \
--bins "$RUNNER_TEMP/arranged" \
--out "$RUNNER_TEMP/npm-dist"
Expand Down
14 changes: 7 additions & 7 deletions .github/workflows/server_rust_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@ on:
push:
branches: ['main']
paths:
- 'packages/server-rust/**'
- 'packages/durable-streams-rust/**'
- '.github/workflows/server_rust_tests.yml'
pull_request:
paths:
- 'packages/server-rust/**'
- 'packages/durable-streams-rust/**'
- '.github/workflows/server_rust_tests.yml'

permissions:
Expand All @@ -24,7 +24,7 @@ jobs:
runs-on: blacksmith-4vcpu-ubuntu-2404
defaults:
run:
working-directory: packages/server-rust
working-directory: packages/durable-streams-rust
steps:
- uses: actions/checkout@v4

Expand All @@ -34,7 +34,7 @@ jobs:

- uses: Swatinem/rust-cache@v2
with:
workspaces: packages/server-rust
workspaces: packages/durable-streams-rust

- name: Build
run: cargo build --release
Expand All @@ -49,7 +49,7 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: durable-streams-server
path: packages/server-rust/target/release/durable-streams-server
path: packages/durable-streams-rust/target/release/durable-streams-server
retention-days: 1

conformance:
Expand Down Expand Up @@ -82,10 +82,10 @@ jobs:
uses: actions/download-artifact@v4
with:
name: durable-streams-server
path: packages/server-rust/target/release
path: packages/durable-streams-rust/target/release

- name: Make binary executable
run: chmod +x packages/server-rust/target/release/durable-streams-server
run: chmod +x packages/durable-streams-rust/target/release/durable-streams-server

- name: Run conformance (${{ matrix.config.name }})
env:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@
#
# Built multi-arch (linux/amd64 + linux/arm64) natively per arch by
# .github/workflows/docker_multiarch_image.yml — the build context is the repo
# root and this Dockerfile is `packages/server-rust/Dockerfile`.
# root and this Dockerfile is `packages/durable-streams-rust/Dockerfile`.

# ---- build stage: compile the release binary (glibc, matches the runtime) ----
FROM rust:1-bookworm AS build
WORKDIR /app
# Copy only what the build needs (no target/, no npm/) so we don't depend on a
# .dockerignore at the shared repo root.
COPY packages/server-rust/Cargo.toml packages/server-rust/Cargo.lock ./
COPY packages/server-rust/src ./src
COPY packages/durable-streams-rust/Cargo.toml packages/durable-streams-rust/Cargo.lock ./
COPY packages/durable-streams-rust/src ./src
# Default features only (no `tier`/`telemetry`) — minimal image, matching the
# conformance matrix. To ship S3 tiering, add `--features tier` here AND
# `ca-certificates` to the runtime stage.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ runs it directly.
features. Builds on Linux and macOS, x64 and arm64.

```bash
# build (run from packages/server-rust)
# build (run from packages/durable-streams-rust)
cargo build --release # → ./target/release/durable-streams-server
cargo test --release # unit + integration tests (protocol conformance: see Conformance below)

Expand Down Expand Up @@ -171,7 +171,7 @@ The instrumentation is deliberately lean and aimed at finding bottlenecks: a sin
```bash
# start the server with a short long-poll timeout to match the suite, then:
RUST_SERVER_URL=http://localhost:4562 pnpm exec vitest run \
--config packages/server-rust/conformance/vitest.config.ts
--config packages/durable-streams-rust/conformance/vitest.config.ts
```

The core protocol suite passes.
Expand Down Expand Up @@ -210,11 +210,4 @@ publishing is re-enabled.

Numbers from **[ds-bench](https://github.com/electric-sql/ds-bench)**, a reproducible single-node harness. All figures below are the default **`wal` mode** (group-commit fsync, resident tail cache off). One server node (`c4d-standard-16-lssd`) pinned to **4 CPUs**, a Kubernetes client fleet driving 256-byte binary appends. Throughput is the saturation ceiling; latency is fleet-wide p50 / p99; memory is the pod cgroup working set (anon + active page cache), peak / p50.

**Writes** — peaks at **~0.86M append/s** at 4 CPUs, scales cleanly to **100k streams**, with median append latency staying sub-ms → ~1.5 ms. Memory tracks **stream count, not bytes** (each stream is a lean record plus its open file; data lives on disk / in the page cache, never resident), so it stays in tens–hundreds of MiB even at 100k streams, with p50 ≪ peak.

| streams | append/s | latency p50 / p99 (ms) | memory peak / p50 (MiB) |
| ------- | -------- | ---------------------- | ----------------------- |
| 100 | 520k | 0.26 / 0.46 | 103 / 45 |
| 1 000 | 650k | 1.26 / 6.4 | 52 / 41 |
| 10 000 | 572k | 1.47 / 203 | 202 / 177 |
| 100 000 | **860k** | — | 950 / 515 |
**Writes** — peaks at **860,000 append/s** at 4 CPUs, scales cleanly to **100k streams**, with median append latency staying sub-ms → ~1.5 ms. Memory tracks **stream count, not bytes** (each stream is a lean record plus its open file; data lives on disk / in the page cache, never resident), so it stays in tens–hundreds of MiB even at 100k streams, with p50 ≪ peak.
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@
*
* Two modes:
* - CI / default: builds nothing, but spawns the release binary
* (packages/server-rust/target/release/durable-streams-server) itself,
* (packages/durable-streams-rust/target/release/durable-streams-server) itself,
* mirroring the Caddy harness. Run with: `pnpm vitest run --project server-rust`
* (build the binary first with `cargo build --release`).
* - Manual: set RUST_SERVER_URL to point at an already-running server, e.g.
* RUST_SERVER_URL=http://localhost:4562 vitest run \
* --config packages/server-rust/conformance/vitest.config.ts
* --config packages/durable-streams-rust/conformance/vitest.config.ts
*/
import { spawn } from 'node:child_process'
import { mkdtempSync } from 'node:fs'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ npm install -g @electric-ax/durable-streams-server-rust
durable-streams-server --port 4438 --data-dir ./data
```

The correct prebuilt binary for your platform is installed automatically as an optional dependency (Linux/macOS, x64/arm64). See the [server README](https://github.com/durable-streams/durable-streams/tree/main/packages/server-rust) for usage and flags.
The correct prebuilt binary for your platform is installed automatically as an optional dependency (Linux/macOS, x64/arm64). See the [server README](https://github.com/durable-streams/durable-streams/tree/main/packages/durable-streams-rust) for usage and flags.

Licensed under Apache-2.0.
Original file line number Diff line number Diff line change
Expand Up @@ -13,51 +13,51 @@ import { fileURLToPath } from 'node:url'
const here = dirname(fileURLToPath(import.meta.url))

function readJson(p) {
return JSON.parse(readFileSync(p, 'utf8'))
return JSON.parse(readFileSync(p, `utf8`))
}

export function assemble({ version, binsDir, outDir }) {
if (!version) throw new Error('assemble: version is required')
const targets = readJson(join(here, 'targets.json'))
const mainTpl = readJson(join(here, 'templates', 'main.package.json'))
const platTpl = readJson(join(here, 'templates', 'platform.package.json'))
if (!version) throw new Error(`assemble: version is required`)
const targets = readJson(join(here, `targets.json`))
const mainTpl = readJson(join(here, `templates`, `main.package.json`))
const platTpl = readJson(join(here, `templates`, `platform.package.json`))

// Platform packages.
const optionalDependencies = {}
for (const t of targets) {
const src = join(binsDir, t.rustTarget, 'durable-streams-server')
const src = join(binsDir, t.rustTarget, `durable-streams-server`)
if (!existsSync(src))
throw new Error(`assemble: missing binary for ${t.rustTarget} at ${src}`)
const pkgDir = join(outDir, t.rustTarget)
rmSync(pkgDir, { recursive: true, force: true })
mkdirSync(join(pkgDir, 'bin'), { recursive: true })
const dest = join(pkgDir, 'bin', 'durable-streams-server')
mkdirSync(join(pkgDir, `bin`), { recursive: true })
const dest = join(pkgDir, `bin`, `durable-streams-server`)
copyFileSync(src, dest)
chmodSync(dest, 0o755)

const pj = { ...platTpl, name: t.pkg, version, os: [t.os], cpu: [t.cpu] }
if (t.libc) pj.libc = [t.libc]
writeFileSync(
join(pkgDir, 'package.json'),
JSON.stringify(pj, null, 2) + '\n'
join(pkgDir, `package.json`),
JSON.stringify(pj, null, 2) + `\n`
)
optionalDependencies[t.pkg] = version
}

// Main package.
const mainDir = join(outDir, 'main')
const mainDir = join(outDir, `main`)
rmSync(mainDir, { recursive: true, force: true })
mkdirSync(join(mainDir, 'bin'), { recursive: true })
mkdirSync(join(mainDir, `bin`), { recursive: true })
copyFileSync(
join(here, 'bin', 'launcher.cjs'),
join(mainDir, 'bin', 'launcher.cjs')
join(here, `bin`, `launcher.cjs`),
join(mainDir, `bin`, `launcher.cjs`)
)
copyFileSync(join(here, 'targets.json'), join(mainDir, 'targets.json'))
copyFileSync(join(here, 'README.md'), join(mainDir, 'README.md'))
copyFileSync(join(here, `targets.json`), join(mainDir, `targets.json`))
copyFileSync(join(here, `README.md`), join(mainDir, `README.md`))
const mainPj = { ...mainTpl, version, optionalDependencies }
writeFileSync(
join(mainDir, 'package.json'),
JSON.stringify(mainPj, null, 2) + '\n'
join(mainDir, `package.json`),
JSON.stringify(mainPj, null, 2) + `\n`
)

return {
Expand All @@ -73,7 +73,7 @@ export function assemble({ version, binsDir, outDir }) {
function parseArgs(argv) {
const out = {}
for (let i = 0; i < argv.length; i += 2) {
const k = argv[i].replace(/^--/, '')
const k = argv[i].replace(/^--/, ``)
out[k] = argv[i + 1]
}
return out
Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
#!/usr/bin/env node
'use strict'
const path = require('node:path')
const { execFileSync } = require('node:child_process')
const path = require(`node:path`)
const { execFileSync } = require(`node:child_process`)

const targets = require('../targets.json')
const targets = require(`../targets.json`)
const key = `${process.platform}-${process.arch}`
const target = targets.find((t) => t.node === key)

if (!target) {
const supported = targets.map((t) => t.node).join(', ')
const supported = targets.map((t) => t.node).join(`, `)
console.error(
`durable-streams-server: unsupported platform "${key}". Supported: ${supported}.`
)
Expand All @@ -19,7 +19,7 @@ let binary
try {
// Resolve via package.json (always resolvable) then join the known binary path.
const pkgJson = require.resolve(`${target.pkg}/package.json`)
binary = path.join(path.dirname(pkgJson), 'bin', 'durable-streams-server')
binary = path.join(path.dirname(pkgJson), `bin`, `durable-streams-server`)
} catch {
console.error(
`durable-streams-server: the platform package "${target.pkg}" is not installed.\n` +
Expand All @@ -30,8 +30,8 @@ try {
}

try {
execFileSync(binary, process.argv.slice(2), { stdio: 'inherit' })
execFileSync(binary, process.argv.slice(2), { stdio: `inherit` })
} catch (err) {
// execFileSync throws on non-zero exit; mirror the child's exit code.
process.exit(typeof err.status === 'number' ? err.status : 1)
process.exit(typeof err.status === `number` ? err.status : 1)
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"repository": {
"type": "git",
"url": "https://github.com/durable-streams/durable-streams.git",
"directory": "packages/server-rust"
"directory": "packages/durable-streams-rust"
},
"bin": { "durable-streams-server": "bin/launcher.cjs" },
"files": ["bin/launcher.cjs", "targets.json", "README.md"],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"repository": {
"type": "git",
"url": "https://github.com/durable-streams/durable-streams.git",
"directory": "packages/server-rust"
"directory": "packages/durable-streams-rust"
},
"files": ["bin/durable-streams-server"],
"os": [],
Expand Down
Loading
Loading