Skip to content

Commit c5d098b

Browse files
Address final Cosmos3 modular feedback
1 parent e3b2f86 commit c5d098b

11 files changed

Lines changed: 1499 additions & 935 deletions

File tree

docs/source/en/api/pipelines/cosmos3.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -750,6 +750,7 @@ pipe = Cosmos3OmniModularPipeline.from_pretrained(
750750
"nvidia/Cosmos3-Nano", torch_dtype=torch.bfloat16
751751
)
752752
pipe.load_components(torch_dtype=torch.bfloat16)
753+
pipe.enable_safety_checker()
753754

754755
videos = pipe(
755756
prompt='{"scene":"A robot arm in a kitchen"}',
@@ -772,6 +773,7 @@ from diffusers import ModularPipeline
772773

773774
pipe = ModularPipeline.from_pretrained("nvidia/Cosmos3-Nano", torch_dtype=torch.bfloat16)
774775
pipe.load_components(torch_dtype=torch.bfloat16)
776+
pipe.enable_safety_checker()
775777
videos = pipe(
776778
prompt='{"scene":"A robot arm in a kitchen"}', num_frames=1, height=720, width=1280, output="videos"
777779
)
@@ -797,6 +799,7 @@ from diffusers.utils import encode_video, export_to_video, load_image, load_vide
797799

798800
pipe = Cosmos3OmniModularPipeline.from_pretrained("nvidia/Cosmos3-Nano", torch_dtype=torch.bfloat16)
799801
pipe.load_components(torch_dtype=torch.bfloat16)
802+
pipe.enable_safety_checker()
800803
pipe.to("cuda")
801804
pipe.scheduler = UniPCMultistepScheduler.from_config(
802805
pipe.scheduler.config, flow_shift=10.0, use_karras_sigmas=False

src/diffusers/modular_pipelines/cosmos/after_decode.py

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,26 @@ def description(self) -> str:
1616
@property
1717
def inputs(self) -> list[InputParam]:
1818
return [
19-
InputParam(name="action_latents", default=None),
20-
InputParam(name="action_mode", default=None),
21-
InputParam(name="raw_action_dim_resolved", default=None),
19+
InputParam(
20+
name="action_latents",
21+
type_hint=torch.Tensor,
22+
default=None,
23+
description="Denoised action latents.",
24+
),
25+
InputParam(
26+
name="action_mode", type_hint=str, default=None, description="Requested action-generation mode."
27+
),
28+
InputParam(
29+
name="raw_action_dim_resolved",
30+
type_hint=int,
31+
default=None,
32+
description="Unpadded action-vector dimension.",
33+
),
2234
]
2335

2436
@property
2537
def intermediate_outputs(self) -> list[OutputParam]:
26-
return [OutputParam("action")]
38+
return [OutputParam("action", type_hint=list[torch.Tensor], description="Generated action vectors.")]
2739

2840
@torch.no_grad()
2941
def __call__(self, components: Cosmos3OmniModularPipeline, state: PipelineState) -> PipelineState:
@@ -53,16 +65,31 @@ def description(self) -> str:
5365
@property
5466
def inputs(self) -> list[InputParam]:
5567
return [
56-
InputParam(name="videos", required=True),
57-
InputParam(name="output_path", required=True),
58-
InputParam(name="fps", type_hint=float, default=24.0),
59-
InputParam(name="sound", default=None),
60-
InputParam(name="sampling_rate", default=None),
68+
InputParam(name="videos", required=True, description="Generated video frames to export."),
69+
InputParam(
70+
name="output_path",
71+
type_hint=str,
72+
required=True,
73+
description="Destination path for the exported video.",
74+
),
75+
InputParam(name="fps", type_hint=float, default=24.0, description="Frame rate of the exported video."),
76+
InputParam(
77+
name="sound",
78+
type_hint=torch.Tensor,
79+
default=None,
80+
description="Generated waveform to mux into the video.",
81+
),
82+
InputParam(
83+
name="sampling_rate",
84+
type_hint=int,
85+
default=None,
86+
description="Sample rate of the generated waveform in Hz.",
87+
),
6188
]
6289

6390
@property
6491
def intermediate_outputs(self) -> list[OutputParam]:
65-
return [OutputParam("output_path")]
92+
return [OutputParam("output_path", type_hint=str, description="Path of the exported video file.")]
6693

6794
@torch.no_grad()
6895
def __call__(self, components: Cosmos3OmniModularPipeline, state: PipelineState) -> PipelineState:

0 commit comments

Comments
 (0)