diff --git a/.gitignore b/.gitignore index 481706a..6ef0ecb 100644 --- a/.gitignore +++ b/.gitignore @@ -161,6 +161,7 @@ cython_debug/ *.ckpt *.wav +*.asd wandb/* *.out diff --git a/optimized/mlx/ableton/AudioInserter/AudioInserter.py b/optimized/mlx/ableton/AudioInserter/AudioInserter.py new file mode 100644 index 0000000..5294243 --- /dev/null +++ b/optimized/mlx/ableton/AudioInserter/AudioInserter.py @@ -0,0 +1,149 @@ +import Live +import socket +import threading +import os +import json +import re + +from _Framework.ControlSurface import ControlSurface + +SOCKET_HOST = "127.0.0.1" +SOCKET_PORT = 9129 +BUFFER_SIZE = 4096 + + +class AudioInserter(ControlSurface): + + def __init__(self, c_instance): + super().__init__(c_instance) + self._c_instance = c_instance + self._server_thread = None + self._running = False + self._start_server() + self.log_message("AudioInserter: started, listening on port %d" % SOCKET_PORT) + + def _start_server(self): + self._running = True + self._server_thread = threading.Thread(target=self._server_loop, daemon=True) + self._server_thread.start() + + def _server_loop(self): + try: + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind((SOCKET_HOST, SOCKET_PORT)) + srv.listen(5) + srv.settimeout(1.0) + while self._running: + try: + conn, _ = srv.accept() + data = conn.recv(BUFFER_SIZE).decode("utf-8").strip() + response = self._handle_command(data) + conn.sendall((response + "\n").encode("utf-8")) + conn.close() + except socket.timeout: + continue + except Exception as e: + self.log_message("AudioInserter socket error: %s" % str(e)) + except Exception as e: + self.log_message("AudioInserter server error: %s" % str(e)) + + def _handle_command(self, raw): + try: + cmd = json.loads(raw) + except Exception: + return json.dumps({"ok": False, "error": "Invalid JSON"}) + + action = cmd.get("action", "") + + if action == "ping": + return json.dumps({"ok": True, "message": "pong"}) + + if action == "insert_audio": + file_path = cmd.get("file_path", "") + if not file_path: + return json.dumps({"ok": False, "error": "No file_path provided"}) + if not os.path.isfile(file_path): + return json.dumps({"ok": False, "error": "File not found: %s" % file_path}) + + result = {"done": False, "ok": False, "error": None} + event = threading.Event() + track_name = cmd.get("track_name", "") + + def do_insert(): + try: + result["ok"], result["error"] = self._insert_audio(file_path, track_name) + except Exception as e: + result["ok"] = False + result["error"] = str(e) + finally: + result["done"] = True + event.set() + + self.schedule_message(1, do_insert) + event.wait(timeout=10.0) + + if result["ok"]: + msg = result.get("error") or ("Inserted: %s" % os.path.basename(file_path)) + return json.dumps({"ok": True, "message": msg}) + else: + return json.dumps({"ok": False, "error": result.get("error", "Unknown error")}) + + return json.dumps({"ok": False, "error": "Unknown action: %s" % action}) + + def _clean_track_name(self, name): + name = re.sub(r'\b\w+\s*:\s*\S+', '', name) + name = re.sub(r'\s{2,}', ' ', name).strip() + return name if name else "Audio" + + def _insert_audio(self, file_path, track_name=""): + import shutil, time + song = self._c_instance.song() + insert_time = song.current_song_time + + song.create_audio_track(-1) + track = song.tracks[-1] + + raw_name = track_name if track_name else os.path.splitext(os.path.basename(file_path))[0] + clean_name = self._clean_track_name(raw_name) + track.name = clean_name + + # Copy file to a unique path so each generation is preserved independently + ext = os.path.splitext(file_path)[1] + unique_name = "%s_%s%s" % (clean_name.replace(" ", "_"), int(time.time()), ext) + dest_dir = os.path.dirname(file_path) + unique_path = os.path.join(dest_dir, unique_name) + shutil.copy2(file_path, unique_path) + self.log_message("AudioInserter: copied to %s" % unique_path) + + # Step 1: create audio clip in session slot 0 + slot = track.clip_slots[0] + slot.create_audio_clip(unique_path) + clip = slot.clip + + if clip is None: + return False, "clip is None after create_audio_clip" + + clip.name = clean_name + + self.log_message("AudioInserter: session clip created, length=%s is_session_clip=%s" % ( + clip.length, clip.is_session_clip)) + + # Step 2: duplicate to arrangement at playhead using track.duplicate_clip_to_arrangement + # Signature from Live 11 sources: duplicate_clip_to_arrangement(clip, position) + try: + track.duplicate_clip_to_arrangement(clip, insert_time) + self.log_message("AudioInserter: duplicate_clip_to_arrangement succeeded") + + # Step 3: delete the session clip — we only want it in arrangement + slot.delete_clip() + + return True, None + except Exception as e: + self.log_message("AudioInserter: duplicate_clip_to_arrangement failed: %s" % str(e)) + return False, "duplicate_clip_to_arrangement failed: %s" % str(e) + + def disconnect(self): + self._running = False + super().disconnect() + self.log_message("AudioInserter: disconnected") \ No newline at end of file diff --git a/optimized/mlx/ableton/AudioInserter/__init__.py b/optimized/mlx/ableton/AudioInserter/__init__.py new file mode 100644 index 0000000..376795e --- /dev/null +++ b/optimized/mlx/ableton/AudioInserter/__init__.py @@ -0,0 +1,4 @@ +from .AudioInserter import AudioInserter + +def create_instance(c_instance): + return AudioInserter(c_instance) diff --git a/optimized/mlx/ableton/README.md b/optimized/mlx/ableton/README.md new file mode 100644 index 0000000..3376cc2 --- /dev/null +++ b/optimized/mlx/ableton/README.md @@ -0,0 +1,66 @@ +# Ableton Live integration (Experimental!) + +Insert `sa3` generations directly into Ableton Live at the current playhead position. + +Two pieces work together: + +| File | Where it runs | +|------|--------------| +| `AudioInserter/` | Inside Ableton — a MIDI Remote Script that listens on a local socket | +| `insert_audio.py` | On the command line — sends files to that socket | + +## 1. Install the AudioInserter Remote Script + +There are two places Ableton looks for Remote Scripts on Mac: + +| Location | Notes | +|----------|-------| +| `~/Music/Ableton/User Library/Remote Scripts/` | **Recommended** — persists across Ableton updates | +| `/Applications/Ableton Live *.app/Contents/App-Resources/MIDI Remote Scripts/` | Works, but gets wiped when you update Ableton | + +Install into the User Library (create the folder if it doesn't exist yet): + +```bash +mkdir -p ~/Music/Ableton/User\ Library/Remote\ Scripts +cp -r AudioInserter ~/Music/Ableton/User\ Library/Remote\ Scripts/AudioInserter +``` + +Then in Ableton: **Preferences → MIDI → Control Surfaces** — add a new surface and choose **AudioInserter**. No MIDI port needed. + +Verify it's running: + +```bash +python3 insert_audio.py --ping +# ✓ Ableton is connected and ready +``` + +## 2. Insert audio + +```bash +# Insert a specific file at the current playhead +python3 insert_audio.py /path/to/out.wav + +# Insert whatever wav was most recently dropped on your Desktop +python3 insert_audio.py + +# Watch a folder and auto-insert each new file as it appears +python3 insert_audio.py --watch ~/Desktop +``` + +The track is named after the file by default. If the env var `_SA3_LAST_PROMPT` is set, that prompt text is used as the track name instead — handy if you wrap `sa3` in a shell alias. + +## 3. Typical workflow with sa3 + +```bash +# Generate +./sa3 --prompt "driving techno loop" --dit medium --decoder same-l --out out.wav + +# Insert +python3 ableton/insert_audio.py out.wav +``` + +Or use watch mode: start it once and every `out.wav` you generate lands in Ableton automatically: + +```bash +python3 ableton/insert_audio.py --watch /path/to/sa3/output/dir +``` diff --git a/optimized/mlx/ableton/insert_audio.py b/optimized/mlx/ableton/insert_audio.py new file mode 100644 index 0000000..bf6bf67 --- /dev/null +++ b/optimized/mlx/ableton/insert_audio.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +""" +insert_audio.py — CLI tool to insert audio files into Ableton Live. + +Usage: + python insert_audio.py # insert latest file in ~/Desktop + python insert_audio.py /path/to/file.wav # insert specific file + python insert_audio.py --watch ~/renders # watch folder, auto-insert on new file + python insert_audio.py --ping # check if Ableton is connected + +Options: + --dir DIR Directory to search for latest file (default: ~/Desktop) + --ext EXT File extension filter, e.g. wav, aiff, mp3 (default: wav aiff mp3 flac) + --watch Watch mode: monitor DIR and auto-insert new files as they appear + --ping Just check if the Ableton script is running +""" + +import sys +import os +import json +import socket +import argparse +import time + +HOST = "127.0.0.1" +PORT = 9129 +TIMEOUT = 5.0 +AUDIO_EXTENSIONS = {".wav", ".aiff", ".aif", ".mp3", ".flac", ".ogg", ".m4a"} + + +# ───────────────────────────────────────────── +# Socket communication +# ───────────────────────────────────────────── + +def send_command(cmd: dict) -> dict: + """Send a JSON command to the Ableton Remote Script and return the response.""" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(TIMEOUT) + s.connect((HOST, PORT)) + s.sendall((json.dumps(cmd) + "\n").encode("utf-8")) + response = b"" + while True: + chunk = s.recv(4096) + if not chunk: + break + response += chunk + if b"\n" in chunk: + break + s.close() + return json.loads(response.decode("utf-8").strip()) + except ConnectionRefusedError: + return {"ok": False, "error": "Cannot connect to Ableton. Make sure:\n" + " 1. Ableton Live is open\n" + " 2. AudioInserter is selected in Preferences → MIDI → Control Surface"} + except socket.timeout: + return {"ok": False, "error": "Ableton timed out — is a project open?"} + except Exception as e: + return {"ok": False, "error": str(e)} + + +def ping(): + result = send_command({"action": "ping"}) + if result.get("ok"): + print("✓ Ableton is connected and ready") + return True + else: + print("✗ " + result.get("error", "Unknown error")) + return False + + +def insert_file(file_path: str) -> bool: + abs_path = os.path.abspath(file_path) + if not os.path.isfile(abs_path): + print(f"✗ File not found: {abs_path}") + return False + + print(f"→ Inserting: {os.path.basename(abs_path)}") + track_name = os.environ.get("_SA3_LAST_PROMPT", "") + result = send_command({"action": "insert_audio", "file_path": abs_path, "track_name": track_name}) + + if result.get("ok"): + msg = result.get("message", "Done") + print(f"✓ {msg}") + return True + else: + print(f"✗ {result.get('error', 'Unknown error')}") + return False + + +# ───────────────────────────────────────────── +# File discovery +# ───────────────────────────────────────────── + +def find_latest_audio(directory: str, extensions: set) -> str | None: + """Return the most recently modified audio file in the given directory.""" + directory = os.path.expanduser(directory) + if not os.path.isdir(directory): + print(f"✗ Directory not found: {directory}") + return None + + candidates = [] + for f in os.listdir(directory): + ext = os.path.splitext(f)[1].lower() + if ext in extensions: + full = os.path.join(directory, f) + candidates.append((os.path.getmtime(full), full)) + + if not candidates: + exts = " ".join(sorted(extensions)) + print(f"✗ No audio files found in {directory}\n (looking for: {exts})") + return None + + candidates.sort(reverse=True) + return candidates[0][1] + + +# ───────────────────────────────────────────── +# Watch mode +# ───────────────────────────────────────────── + +def watch_directory(directory: str, extensions: set): + """Monitor a directory and auto-insert new audio files as they appear.""" + directory = os.path.expanduser(directory) + if not os.path.isdir(directory): + print(f"✗ Directory not found: {directory}") + sys.exit(1) + + print(f"👁 Watching {directory} for new audio files... (Ctrl+C to stop)\n") + + # Record existing files so we don't re-insert them on start + seen = set() + for f in os.listdir(directory): + if os.path.splitext(f)[1].lower() in extensions: + seen.add(os.path.join(directory, f)) + + try: + while True: + time.sleep(0.75) + for f in os.listdir(directory): + ext = os.path.splitext(f)[1].lower() + if ext not in extensions: + continue + full = os.path.join(directory, f) + if full not in seen: + seen.add(full) + # Short wait to ensure file write is complete + time.sleep(0.3) + print(f"\n[{time.strftime('%H:%M:%S')}] New file detected") + insert_file(full) + except KeyboardInterrupt: + print("\n\nStopped watching.") + + +# ───────────────────────────────────────────── +# CLI entry point +# ───────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser( + description="Insert audio files into Ableton Live at the current playhead position.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument( + "file", nargs="?", help="Audio file to insert (omit to use latest file in --dir)" + ) + parser.add_argument( + "--dir", default="~/Desktop", help="Directory to search for latest audio file" + ) + parser.add_argument( + "--ext", nargs="+", default=None, + help="File extensions to consider, e.g. --ext wav aiff" + ) + parser.add_argument( + "--watch", action="store_true", + help="Watch --dir and auto-insert new files as they appear" + ) + parser.add_argument( + "--ping", action="store_true", + help="Check if Ableton is running and the script is active" + ) + + args = parser.parse_args() + + # Build extension set + if args.ext: + extensions = {"." + e.lstrip(".").lower() for e in args.ext} + else: + extensions = AUDIO_EXTENSIONS + + # ── Ping ── + if args.ping: + sys.exit(0 if ping() else 1) + + # ── Watch mode ── + if args.watch: + watch_directory(args.dir, extensions) + return + + # ── Single insert ── + if args.file: + file_path = args.file + else: + file_path = find_latest_audio(args.dir, extensions) + if not file_path: + sys.exit(1) + print(f"Latest file: {os.path.basename(file_path)}") + + success = insert_file(file_path) + sys.exit(0 if success else 1) + + +if __name__ == "__main__": + main() diff --git a/optimized/mlx/scripts/sa3_mlx.py b/optimized/mlx/scripts/sa3_mlx.py index 36bf57b..c8893b6 100644 --- a/optimized/mlx/scripts/sa3_mlx.py +++ b/optimized/mlx/scripts/sa3_mlx.py @@ -238,26 +238,40 @@ def save_wav(path: str, audio: np.ndarray, sample_rate: int = SAMPLE_RATE): def read_wav(path: str) -> np.ndarray: - """Read a 16-bit PCM WAV at 44.1 kHz. Returns (2, T) float32 in [-1, 1]. + """Read a WAV file. Returns (2, T) float32 in [-1, 1]. - Mono input is duplicated to stereo. Other sample rates / bit-depths error out - with a hint to ffmpeg-resample first — keeps this script dependency-free. + Handles 16-bit PCM at 44.1 kHz natively. Falls back to ffmpeg for any + other format (24-bit, 32-bit float, 48 kHz, etc.) + Mono input is duplicated to stereo. """ - with wave.open(path, "rb") as w: - nch, sw, sr, nframes = w.getnchannels(), w.getsampwidth(), w.getframerate(), w.getnframes() - if sr != SAMPLE_RATE: - raise ValueError( - f"{path}: sample rate {sr} Hz; need {SAMPLE_RATE}. " - f"Resample first: ffmpeg -i {path} -ar {SAMPLE_RATE} -ac 2 -sample_fmt s16 out.wav" - ) - if sw != 2: - raise ValueError(f"{path}: {sw*8}-bit WAV; need 16-bit PCM. Convert with ffmpeg.") - raw = np.frombuffer(w.readframes(nframes), dtype=np.int16).astype(np.float32) / 32767.0 - if nch == 1: - audio = np.stack([raw, raw], axis=0) # (2, T) - else: - audio = raw.reshape(-1, nch).T[:2] # (2, T) - return audio + try: + with wave.open(path, "rb") as w: + nch, sw, sr, nframes = w.getnchannels(), w.getsampwidth(), w.getframerate(), w.getnframes() + if sr == SAMPLE_RATE and sw == 2: + raw = np.frombuffer(w.readframes(nframes), dtype=np.int16).astype(np.float32) / 32767.0 + if nch == 1: + return np.stack([raw, raw], axis=0) # (2, T) + return raw.reshape(-1, nch).T[:2] # (2, T) + except wave.Error: + pass # unsupported format (32-bit float, 24-bit PCM, etc.) + + # Fallback: decode via ffmpeg — handles any sample rate, bit depth, or format + try: + result = subprocess.run( + ["ffmpeg", "-v", "error", "-i", path, + "-f", "s16le", "-ar", str(SAMPLE_RATE), "-ac", "2", "-"], + capture_output=True, check=True, + ) + except FileNotFoundError: + raise RuntimeError( + f"{path}: unsupported WAV format. Install ffmpeg to handle 24-bit/32-bit/48kHz audio:\n" + f" brew install ffmpeg\n" + f"Or convert manually: ffmpeg -i \"{path}\" -ar {SAMPLE_RATE} -ac 2 -sample_fmt s16 resampled.wav" + ) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"{path}: ffmpeg failed — {e.stderr.decode().strip()}") + raw = np.frombuffer(result.stdout, dtype=np.int16).astype(np.float32) / 32767.0 + return raw.reshape(-1, 2).T def load_encoder(decoder_name: str, dtype): @@ -420,11 +434,11 @@ def main(): "script doesn't). Default: on.") # ── Output ──────────────────────────────────────────────────────────────── - ap.add_argument("--out", default="out.wav", + ap.add_argument("--out", default=None, help="Output WAV path. Relative paths land in the project's output/ " "directory (auto-created); absolute paths are used as-is. " "Always written as 16-bit PCM stereo at 44.1 kHz, trimmed to " - "exactly --seconds.") + "exactly --seconds. If omitted, auto-named from the prompt and seed.") ap.add_argument("--play", action="store_true", help="After writing the WAV, play it through the default output device " "via the macOS `afplay` binary. Blocking — the script exits when " @@ -434,17 +448,20 @@ def main(): if args.steps < 1: ap.error(f"--steps must be ≥ 1 (got {args.steps})") - # Relative --out paths land in the project's output/ folder; absolute paths - # are honoured as-is so users can still write anywhere on disk. + args = prompt_user_if_missing(args) + if args.prompt is None: + args.prompt = input("Prompt: ").strip() + + # Resolve output path — auto-name from prompt+seed when --out is not given. + if args.out is None: + import re as _re + slug = _re.sub(r'[^a-z0-9]+', '_', args.prompt.lower()).strip('_')[:48] + args.out = f"{slug}_{args.seed}.wav" if slug else f"out_{args.seed}.wav" out_path = Path(args.out) if not out_path.is_absolute(): out_path = REPO / "output" / out_path out_path.parent.mkdir(parents=True, exist_ok=True) args.out = str(out_path) - - args = prompt_user_if_missing(args) - if args.prompt is None: - args.prompt = input("Prompt: ").strip() # Empty prompt is allowed — T5Gemma will produce padding-only embeddings, # which (with the learned padding_embedding) is the unconditional case. dtype = mx.float32 if args.dit_dtype == "fp32" else mx.float16