diff --git a/src/pipeline/.env.example b/src/pipeline/.env.example index 8715da6..55511b7 100644 --- a/src/pipeline/.env.example +++ b/src/pipeline/.env.example @@ -20,6 +20,11 @@ SESAME_API_KEY= # Required for Sesame work: the privately hosted CSM-1B deployment URL # (no public default exists; see transports/sesame.ts). SESAME_URL= +SPEECHIFY_API_KEY= +# Speechify clone consent declaration: full name + email of the maintainer +# creating the clone (sent with POST /v1/voices; see transports/speechify.ts). +SPEECHIFY_CONSENT_FULL_NAME= +SPEECHIFY_CONSENT_EMAIL= # Vercel Blob store hosting the arena clips (audio/{hash}.mp3). # Maintainers: pull from the Vercel project for this site. diff --git a/src/pipeline/RUNBOOK.md b/src/pipeline/RUNBOOK.md index 04c5ed3..833fd3f 100644 --- a/src/pipeline/RUNBOOK.md +++ b/src/pipeline/RUNBOOK.md @@ -152,6 +152,7 @@ upload. No TTFB bench (Human has no API; `latencyMs: null`). | Neuphonic | neu_fast | **PARKED, hosted but unregistered (2026-06-12)**: 80 clips uploaded + HEAD-verified under arenaApiId `neu-fast` (FROZEN), but the API ignores the model param, so these clips came from the SAME served pool as neu_hq's; registering both would put two rows on one system (methodology integrity wins). The request body did send `model: neu_fast`, so the clips become retroactively correct if Neuphonic exposes true per-model selection | Revisit if Neuphonic exposes true per-model selection; the pre-registration row stays in pipeline/models.ts | | Hume | Octave, Octave 2 | Clone creation is **Platform-UI only** (the API uses clones but cannot create them), manual console work | `bun run humanness:clone hume` prints the console steps; record ids with `--record`; then steps 2-5. Octave 2 voices are not backward compatible with Octave 1, so clone on the right model family | | Sesame | CSM-1B | **Decision needed**: the hosted deployment's clone path expects source samples staged in its own private storage, which this pipeline cannot reach | Decide: get upload access / self-host CSM-1B cloning / drop Sesame. Transport synthesis works against the SESAME_URL host; `createClone` intentionally throws with this note | +| Speechify | Simba 3.2 | **BLOCKED on clone-model support (2026-07-17)**: transport verified live end to end (synthesis, streaming TTFB surface, clone create + synth + delete round-trip on a throwaway test voice, deleted after). The API strictly validates `model` (a bogus id 400s with the valid list: simba-english, simba-multilingual, simba-3.0, simba-3.2), so there is no Neuphonic-style shared pool. But cloned voices list only simba-english / simba-multilingual; simba-3.2 is limited to eight stock `*_32` shared voices (beatrice/dominic/edmund/geffen/harper/hugh/imogen/wyatt, en-US + en-GB), so the same-voice methodology cannot run yet. Pre-registration row `speechify-simba-3-2` (proposed frozen arenaApiId `simba-3-2`) staged in pipeline/models.ts | Re-probe periodically: create a clone, check whether its `models` list includes simba-3.2 (or the stock-voice gate lifts). When it does: clone the four source voices (`humanness:clone speechify`, needs the consent env vars), then steps 2-5 | Source clips for cloning: the licensed master recordings live at `pipeline/results/source-voices/originals/{clara,emma,godfrey,nelliot}/` @@ -192,9 +193,16 @@ dropped in; the originals supersede them.) the documented batch route `POST /v1/tts` works with this key (200, audio/mpeg, single-probe first-byte ~463 ms); only the legacy `/v1/audio/speech` path 403s (re-probe: `results/probe-xai-http.ts`). -- **Blob store**: `audio/{hash}.mp3`, public access, `addRandomSuffix: - false`. Uploads are idempotent (skip-if-exists); the token's store id must - be the arena audio origin (bkvlbh5qphzaen1w...). +- **Speechify (2026-07-17)**: `POST /v1/audio/speech` returns JSON with + base64 `audio_data` (not raw bytes); request mp3 via `audio_format`. + Cloning is multipart `POST /v1/voices` (`name`, one `sample` file, + `consent` JSON naming the voice owner: `SPEECHIFY_CONSENT_FULL_NAME` / + `SPEECHIFY_CONSENT_EMAIL` in pipeline/.env). Back-to-back requests 429 + quickly (a second request within ~1 s of the first), so keep + `--concurrency 1` and let generateClips' RateLimitedError backoff pace the + run. TTFB benches the chunked HTTP `/v1/audio/stream` route (single July + 2026 probe saw first audio at ~0.9 s; not a bench). Bench voice is the + stock `harper_32` until cloned voices can run simba-3.2. ## TTFB results provenance diff --git a/src/pipeline/models.ts b/src/pipeline/models.ts index 54ed81d..6469dd7 100644 --- a/src/pipeline/models.ts +++ b/src/pipeline/models.ts @@ -90,6 +90,21 @@ const NEW_MODELS: PipelineModel[] = [ name: 'TTS-1.5 Mini', registered: false, }, + { + // BLOCKED (2026-07-17): the Speechify clone API works (create + synth + + // delete round-trip verified live), but cloned voices list only + // simba-english / simba-multilingual — simba-3.2 is limited to eight + // stock `*_32` shared voices, so the same-voice arena run cannot start. + // No clips exist yet, so the frozen ids below are still a proposal. + // Revisit when clones list simba-3.2 (see RUNBOOK status table). + id: 'speechify-simba-3-2', + slug: 'speechify-simba-3-2', + providerId: 'speechify', + arenaApiId: 'simba-3-2', + vendorModelId: 'simba-3.2', + name: 'Simba 3.2', + registered: false, + }, ]; const registryModels = (): PipelineModel[] => diff --git a/src/pipeline/transports/index.ts b/src/pipeline/transports/index.ts index 27547b5..d8af523 100644 --- a/src/pipeline/transports/index.ts +++ b/src/pipeline/transports/index.ts @@ -14,6 +14,7 @@ import { minimax } from './minimax'; import { neuphonic } from './neuphonic'; import { sesame } from './sesame'; import { smallest } from './smallest'; +import { speechify } from './speechify'; import { xai } from './xai'; import type { ProviderTransport } from './types'; @@ -28,6 +29,7 @@ export const TRANSPORTS: Record = { neuphonic, hume, sesame, + speechify, xai, }; diff --git a/src/pipeline/transports/speechify.ts b/src/pipeline/transports/speechify.ts new file mode 100644 index 0000000..9d6d7ea --- /dev/null +++ b/src/pipeline/transports/speechify.ts @@ -0,0 +1,113 @@ +/** + * Speechify transport (Simba family). Request shapes verified live against + * api.speechify.ai on 2026-07-17 (see the RUNBOOK status table): synthesis is + * `POST /v1/audio/speech` returning JSON with base64 `audio_data`; cloning is + * multipart `POST /v1/voices` with a consent declaration (create + synth + + * delete round-trip verified on a throwaway voice); TTFB runs on the chunked + * HTTP `POST /v1/audio/stream` route. The API strictly validates `model` + * (bogus ids 400 with the valid list), so per-model clips are genuine. + * + * Simba 3.2 caveat: cloned voices currently list only simba-english / + * simba-multilingual, so simba-3.2 synthesis is limited to the eight stock + * `*_32` shared voices until Speechify enables clones for it. The bench + * therefore pins a stock 3.2 voice; switch it to the arena Clara clone once + * clones can run 3.2. + */ +import { readFileSync } from 'node:fs'; +import { basename, extname } from 'node:path'; + +import { requireEnv } from '../env'; +import { + BENCH_TEXT, + anyBytesMarker, + decodeBase64Audio, + httpStreamTrial, + postFormForJson, + requestJson, +} from './http'; +import { TransportError, type ProviderTransport, type TtfbPlan } from './types'; + +const API = 'https://api.speechify.ai'; +/** Stock voices for the bench: simba-3.2 has no clone support yet. */ +const BENCH_VOICES: Record = { + 'simba-3.2': 'harper_32', +}; +const DEFAULT_BENCH_VOICE = 'cleon'; + +const apiKey = (): string => requireEnv('SPEECHIFY_API_KEY'); + +export const speechify: ProviderTransport = { + providerId: 'speechify', + apiKeyEnv: 'SPEECHIFY_API_KEY', + + synthesize: async ({ vendorModelId, providerVoiceId, text }) => { + const result = await requestJson<{ audio_data?: string }>( + 'POST', + `${API}/v1/audio/speech`, + { authorization: `Bearer ${apiKey()}` }, + { + input: text, + voice_id: providerVoiceId, + model: vendorModelId, + audio_format: 'mp3', + }, + 'speechify', + ); + if (!result.audio_data) { + throw new TransportError('speechify returned no audio_data'); + } + return { bytes: decodeBase64Audio(result.audio_data), format: 'mp3' as const }; + }, + + createClone: async ({ displayName, sampleFiles }) => { + // The clone endpoint takes a single `sample` file plus a consent + // declaration naming the voice owner (the maintainer running the clone). + const sampleFile = sampleFiles[0]; + const mimeType = extname(sampleFile).toLowerCase() === '.mp3' ? 'audio/mpeg' : 'audio/wav'; + const form = new FormData(); + form.append('name', displayName); + form.append( + 'sample', + new Blob([readFileSync(sampleFile)], { type: mimeType }), + basename(sampleFile), + ); + form.append( + 'consent', + JSON.stringify({ + fullName: requireEnv('SPEECHIFY_CONSENT_FULL_NAME'), + email: requireEnv('SPEECHIFY_CONSENT_EMAIL'), + }), + ); + const result = await postFormForJson<{ id: string }>( + `${API}/v1/voices`, + { authorization: `Bearer ${apiKey()}` }, + form, + 'speechify voices', + ); + return result.id; + }, + + ttfbPlanFor: (vendorModelId): TtfbPlan => ({ + transport: 'http-stream', + notes: + 'chunked HTTP /v1/audio/stream; stock bench voice until clones can run simba-3.2', + trial: () => + httpStreamTrial( + `${API}/v1/audio/stream`, + { + method: 'POST', + headers: { + authorization: `Bearer ${apiKey()}`, + 'content-type': 'application/json', + accept: 'audio/mpeg', + }, + body: JSON.stringify({ + input: BENCH_TEXT, + voice_id: BENCH_VOICES[vendorModelId] ?? DEFAULT_BENCH_VOICE, + model: vendorModelId, + }), + }, + anyBytesMarker, + ), + }), +}; diff --git a/src/pipeline/voices.ts b/src/pipeline/voices.ts index 90b9a99..74abebe 100644 --- a/src/pipeline/voices.ts +++ b/src/pipeline/voices.ts @@ -60,6 +60,7 @@ const STATIC_CLONED_VOICE_IDS: Record = { neuphonic: {}, hume: {}, sesame: {}, + speechify: {}, }; const LOCAL_VOICES_PATH = resolve(import.meta.dir, 'voices.local.json');