Skip to content

Commit ab33c17

Browse files
feat: MuseTalk V15 lip sync skill
Add AI lip sync skill powered by MuseTalk V15 — synchronizes lip movements to match provided audio. Video+audio and image+audio inputs, multi-face support, batch inference. Subprocess isolation for CUDA memory safety. Path traversal protection.
1 parent 4808248 commit ab33c17

3 files changed

Lines changed: 71 additions & 0 deletions

File tree

skills/category/audio.py

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -806,3 +806,51 @@ def register_skills(registry: SkillRegistry) -> None:
806806
"non_commercial", "cc_by_nc",
807807
],
808808
))
809+
810+
# Lip sync — AI-powered lip synchronization (MuseTalk)
811+
registry.register(Skill(
812+
name="lip_sync",
813+
category=SkillCategory.AUDIO,
814+
description=(
815+
"AI lip sync: synchronize a face's lip movements to match "
816+
"provided audio using MuseTalk. Works with both video and image "
817+
"inputs. Supports multiple faces. Requires an audio_path "
818+
"pointing to the speech/voice audio file."
819+
),
820+
parameters=[
821+
SkillParameter(
822+
name="audio_path",
823+
type=ParameterType.STRING,
824+
description="Path to audio file for lip sync (speech/voice)",
825+
required=True,
826+
),
827+
SkillParameter(
828+
name="face_index",
829+
type=ParameterType.INT,
830+
description="Which face to sync: -1 = all faces, 0 = first face, 1 = second, etc.",
831+
required=False,
832+
default=-1,
833+
min_value=-1,
834+
max_value=9,
835+
),
836+
SkillParameter(
837+
name="batch_size",
838+
type=ParameterType.INT,
839+
description="Frames per inference batch (lower = less VRAM)",
840+
required=False,
841+
default=8,
842+
min_value=1,
843+
max_value=32,
844+
),
845+
],
846+
examples=[
847+
"lip_sync:audio_path=/path/to/speech.wav - Sync lips to speech audio",
848+
"lip_sync:audio_path=/path/to/voice.mp3,face_index=0 - Sync first face only",
849+
"lip_sync:audio_path=/path/to/audio.wav,batch_size=4 - Lower VRAM usage",
850+
],
851+
tags=[
852+
"lip", "sync", "lipsync", "dub", "dubbing", "talking", "head",
853+
"face", "voice", "speech", "musetalk", "ai",
854+
],
855+
))
856+

skills/composer.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,14 @@ class SkillComposer:
152152
"synthesize_audio": "generate_audio",
153153
"generate_sound": "generate_audio",
154154
"add_sound": "generate_audio",
155+
# MuseTalk — Lip Sync
156+
"lipsync": "lip_sync",
157+
"dub": "lip_sync",
158+
"dubbing": "lip_sync",
159+
"sync_lips": "lip_sync",
160+
"talking_head": "lip_sync",
161+
"lip_dub": "lip_sync",
162+
"voice_sync": "lip_sync",
155163
}
156164

157165
def __init__(self, registry: Optional[SkillRegistry] = None):
@@ -1240,6 +1248,8 @@ def _get_dispatch() -> dict:
12401248
_f_auto_mask,
12411249
# generate audio (MMAudio)
12421250
_f_generate_audio,
1251+
# lip sync (MuseTalk)
1252+
_f_lip_sync,
12431253
# presets
12441254
_f_fade_to_black, _f_fade_to_white, _f_flash,
12451255
_f_spin, _f_shake, _f_pulse, _f_bounce, _f_drift,
@@ -1419,6 +1429,15 @@ def _get_dispatch() -> dict:
14191429
"synthesize_audio": _f_generate_audio,
14201430
"generate_sound": _f_generate_audio,
14211431
"add_sound": _f_generate_audio,
1432+
# MuseTalk — Lip Sync
1433+
"lip_sync": _f_lip_sync,
1434+
"lipsync": _f_lip_sync,
1435+
"dub": _f_lip_sync,
1436+
"dubbing": _f_lip_sync,
1437+
"sync_lips": _f_lip_sync,
1438+
"talking_head": _f_lip_sync,
1439+
"lip_dub": _f_lip_sync,
1440+
"voice_sync": _f_lip_sync,
14221441
# Presets / Transitions
14231442
"fade_to_black": _f_fade_to_black,
14241443
"fade_to_white": _f_fade_to_white,

skills/handlers/__init__.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@
1515
_f_generate_audio,
1616
)
1717

18+
from .lip_sync import ( # noqa: F401
19+
_f_lip_sync,
20+
)
21+
1822
from .multi_input import ( # noqa: F401
1923
_f_grid,
2024
_f_slideshow,

0 commit comments

Comments
 (0)