From 6c759240db26115934efdbf90411c731a25fe445 Mon Sep 17 00:00:00 2001 From: Davids048 Date: Sun, 10 May 2026 14:41:11 -0700 Subject: [PATCH] [feat] Activate Dreamverse LTX2 integration Apply Dreamverse monorepo changes for stack slice 13/13 from the source branch. Source-Branch: will/dreamverse-monorepo Source-SHA: 03d3e61df69fe99a81eb62d86a3926a5f769f857 Dreamverse-Stack: 13/13 Co-authored-by: SolitaryThinker Co-authored-by: Matthew Noto <99706358+RandNMR73@users.noreply.github.com> Co-authored-by: XOR-op <17672363+XOR-op@users.noreply.github.com> Co-authored-by: Zhang Peiyuan <42993249+jzhang38@users.noreply.github.com> --- docs/contributing/dreamverse-development.md | 100 ++++++++++++++++++++ fastvideo/models/registry.py | 1 + fastvideo/registry.py | 27 ++++-- mkdocs.yml | 6 ++ 4 files changed, 124 insertions(+), 10 deletions(-) create mode 100644 docs/contributing/dreamverse-development.md diff --git a/docs/contributing/dreamverse-development.md b/docs/contributing/dreamverse-development.md new file mode 100644 index 0000000000..6e3ec57463 --- /dev/null +++ b/docs/contributing/dreamverse-development.md @@ -0,0 +1,100 @@ +# Dreamverse Development + +Dreamverse lives under `apps/dreamverse/` as a product app inside the +FastVideo monorepo. Backend code uses the local FastVideo workspace package; +frontend tooling remains standalone under `apps/dreamverse/web/`. + +## Backend tests + +Run CPU-safe backend tests from the FastVideo repository root: + +```bash +uv run --locked --package dreamverse --extra test pytest apps/dreamverse/server/tests/ -m 'not gpu' -q +``` + +## Backend launch + +Launch the migrated backend through the installed console commands: + +```bash +dreamverse-server --port 8009 +dreamverse-mock-server --port 8009 +``` + +If `dreamverse-server` is missing, install FastVideo with the `dreamverse` +extra from the checkout: + +```bash +uv pip install -e ".[dreamverse]" +``` + +## Frontend build and tests + +Run frontend commands from the standalone web app: + +```bash +cd apps/dreamverse/web +pnpm install --frozen-lockfile +pnpm run build +pnpm run test +``` + +Playwright is intentionally run against a live backend as part of the GPU4 +manual verification flow, not in the Phase 3 migration gate. + +## Local GPU4 verification hook + +Use physical GPU 4 for migration smoke tests. `CUDA_VISIBLE_DEVICES=4` makes +that GPU appear as logical GPU 0 inside the process, preserving the previous +Dreamverse deployment behavior. + +```bash +CUDA_VISIBLE_DEVICES=4 dreamverse-server --host 0.0.0.0 --port 8009 +``` + +In another shell, verify the service: + +```bash +curl -s http://localhost:8009/healthz +``` + +Phase 4 adds the public `/healthz`, `/readyz`, `/status`, +`/prompt-system-config`, and `/curated-presets` route coverage needed for the +full Playwright suite. + +## Phase 0 production-equivalent prerequisites + +For the production-equivalent NVFP4 path, install these dependencies +in the FastVideo `.venv` before GPU smoke tests: + +```bash +uv pip install --python .venv/bin/python \ + flashinfer-python flash-attn cerebras-cloud-sdk openai \ + --no-build-isolation +``` + +| Package | Why | +|---|---| +| `flashinfer-python` | Required for NVFP4 quantization. Without it, model load fails with `ImportError: NVFP4 quantization requires flashinfer`. | +| `flash-attn` | Optional but recommended; without it attention falls back to Torch SDPA (functional but slower). | +| `cerebras-cloud-sdk` | Required by the migrated prompt enhancer for the default `cerebras` provider. | +| `openai` | Required by the prompt enhancer's OpenAI-compatible providers + downstream rewrites. | + +### B200 / sm_100a + gcc-15 conda toolchain (flashinfer JIT workaround) + +On hosts where the conda toolchain ships gcc-15 (which nvcc rejects with +`#error -- unsupported GNU version! gcc versions later than 14 are not +supported!`), set these env vars before launching anything that triggers +flashinfer's JIT kernel build: + +```bash +export CC=/usr/bin/gcc-13 +export CXX=/usr/bin/g++-13 +export CUDAHOSTCXX=/usr/bin/g++-13 +export NVCC_PREPEND_FLAGS="-ccbin /usr/bin/gcc-13 -allow-unsupported-compiler" +``` + +`dreamverse-server` does NOT set these — they need to come from the launching +shell. The `dreamverse-deploy` skill +([`.agents/skills/dreamverse-deploy/`](../../.agents/skills/dreamverse-deploy/SKILL.md)) +sets them for you and is the recommended local-deploy path. diff --git a/fastvideo/models/registry.py b/fastvideo/models/registry.py index a298cf6674..a14d0bae5b 100644 --- a/fastvideo/models/registry.py +++ b/fastvideo/models/registry.py @@ -115,6 +115,7 @@ _UPSAMPLERS = { "SRTo720pUpsampler": ("upsamplers", "hunyuan15", "SRTo720pUpsampler"), "SRTo1080pUpsampler": ("upsamplers", "hunyuan15", "SRTo1080pUpsampler"), + "LTX2LatentUpsampler": ("upsamplers", "ltx2_upsampler", "LTX2LatentUpsampler"), } _LEGACY_FAST_VIDEO_MODELS = { diff --git a/fastvideo/registry.py b/fastvideo/registry.py index be1469fe3e..6f7649298c 100644 --- a/fastvideo/registry.py +++ b/fastvideo/registry.py @@ -215,35 +215,42 @@ def _get_config_info( def _register_configs() -> None: - # LTX-2 (base) + # LTX-2 (distilled) — registered FIRST so its detector wins over + # the base detector when both fire. The detector loop in + # ``get_model_name_for_path`` ORs the path-based check with a + # pipeline-name check (``ltx2pipeline``) which the base detector's + # "distilled not in path" predicate matches as True (the + # pipeline_name string contains no "distilled" marker), so the + # less-specific BASE detector would otherwise win when the + # input is the absolute path of the distilled checkpoint. register_configs( sampling_param_cls=None, pipeline_config_cls=LTX2T2VConfig, workload_types=(WorkloadType.T2V, ), hf_model_paths=[ - "Lightricks/LTX-2", - "FastVideo/LTX2-base", - "FastVideo/LTX2-Diffusers", + "FastVideo/LTX2-Distilled-Diffusers", ], model_detectors=[ - lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower()) and "distilled" not in path.lower(), + lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower()) and "distilled" in path.lower(), ], model_family="ltx2", - default_preset="ltx2_base", + default_preset="ltx2_distilled", ) - # LTX-2 (distilled) + # LTX-2 (base) register_configs( sampling_param_cls=None, pipeline_config_cls=LTX2T2VConfig, workload_types=(WorkloadType.T2V, ), hf_model_paths=[ - "FastVideo/LTX2-Distilled-Diffusers", + "Lightricks/LTX-2", + "FastVideo/LTX2-base", + "FastVideo/LTX2-Diffusers", ], model_detectors=[ - lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower()) and "distilled" in path.lower(), + lambda path: ("ltx2" in path.lower() or "ltx-2" in path.lower()) and "distilled" not in path.lower(), ], model_family="ltx2", - default_preset="ltx2_distilled", + default_preset="ltx2_base", ) # Stable Audio Open (text-to-audio). Both variants must be loaded diff --git a/mkdocs.yml b/mkdocs.yml index a9b4b1f1c1..769b6c56d9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -162,6 +162,11 @@ nav: - Design: - Overview: design/overview.md - Training Architecture: design/training_architecture.md + - Server Contracts: + - Overview: design/server_contracts/index.md + - OpenAI HTTP: design/server_contracts/openai.md + - Streaming WebSocket: design/server_contracts/streaming.md + - Dynamo Integration: design/server_contracts/dynamo.md - Developer Guide: - Overview: contributing/overview.md - Developer Environment: @@ -173,6 +178,7 @@ nav: - Testing: contributing/testing.md - Profiling: contributing/profiling.md - Coding Agents: contributing/coding_agents.md + - Dreamverse Development: contributing/dreamverse-development.md - Attention Backend Development: contributing/attention_backend.md - API Reference: - FastVideo: api/fastvideo.md