fix: harden YouTube downloads against yt-dlp 403 failures

maxitoon · maxitoon · commit a4a47ebdc5ee · 2026-02-13T00:24:10.000+01:00
- add multi-strategy yt-dlp retries in Python downloader using fallback format/client profiles\n- add fallback download functions in interactive shell script for audio and video modes\n- improve cleanup pattern for downloaded video extensions\n- add downloader retry test covering first-attempt 403 failure path\n- document YouTube reliability behavior and troubleshooting steps in README
diff --git a/README.md b/README.md
@@ -6,6 +6,7 @@ Local-first audio/video transcription tool powered by [whisper-cli](https://gith
 
 - **Live microphone transcription** with incremental chunk-based output (text appears every ~5 seconds while you speak)
 - **YouTube download + transcribe** — paste a URL, get a transcript
+- **YouTube fallback download strategies** — retries multiple YouTube client profiles to reduce 403 failures
 - **File transcription** — supports Zoom recordings, WhatsApp audio, and any audio/video file
 - **Multiple output formats** — txt, srt, vtt, json
 - **Multi-language support** — English, French, auto-detect
@@ -53,7 +54,23 @@ This launches an interactive menu with these options:
 5. **WhatsApp Audio** — Transcribe a WhatsApp voice message
 6. **Other File** — Transcribe any audio/video file
 
-Transcripts are saved to `~/Desktop/Transcripts/`. Audio downloads are saved to `~/whisper-downloads/` and auto-cleaned after 7 days.
+Transcripts are saved to `~/Documents/Transcripts/`. Audio downloads are saved to `~/whisper-downloads/` and auto-cleaned after 7 days.
+
+## YouTube Download Reliability
+
+When YouTube changes delivery behavior (for example SABR-related client restrictions), a single `yt-dlp` strategy can fail with `HTTP Error 403: Forbidden`.
+
+This project now retries YouTube downloads using multiple fallback profiles:
+
+- Alternative audio/video format selectors
+- Multiple YouTube player clients (`android`, `ios`, `web`)
+- Conservative transfer settings (`--retries`, `--fragment-retries`, `--force-ipv4`)
+
+If you still hit 403 errors, run:
+
+```bash
+yt-dlp -U
+```
 
 ## How Live Transcription Works
 
diff --git a/src/transcriber/downloader.py b/src/transcriber/downloader.py
@@ -2,7 +2,7 @@
 
 import tempfile
 from pathlib import Path
-from typing import Optional
+from typing import Any, Dict, List, Optional
 
 try:
     import yt_dlp
@@ -45,8 +45,8 @@ def download_audio(
         Returns:
             Path to downloaded audio file
         """
-        # Configure yt-dlp options
-        ydl_opts = {
+        # Common yt-dlp options shared across retry strategies.
+        base_opts = {
             "format": f"{quality}/best[height<=480]/best",
             "extract_audio": True,
             "audio_format": format,
@@ -62,29 +62,29 @@ def download_audio(
                 }
             ],
         }
-
-        # Download the video
-        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-            info = ydl.extract_info(url, download=True)
-            filename = ydl.prepare_filename(info)
-
-            # Convert to audio format if needed
-            if info.get("ext") != format:
-                audio_filename = filename.rsplit(".", 1)[0] + f".{format}"
-            else:
-                audio_filename = filename
-
-        audio_path = Path(audio_filename)
-        if not audio_path.exists():
-            # Fallback: look for the file with the expected extension
-            expected_path = Path(filename.rsplit(".", 1)[0] + f".{format}")
-            if expected_path.exists():
-                audio_path = expected_path
-
-        if not audio_path.exists():
-            raise FileNotFoundError(f"Downloaded audio file not found: {audio_path}")
-
-        return audio_path
+        errors: List[str] = []
+        for strategy in self._download_strategies(quality):
+            ydl_opts = dict(base_opts)
+            ydl_opts.update(strategy)
+
+            try:
+                with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                    info = ydl.extract_info(url, download=True)
+                    filename = ydl.prepare_filename(info)
+                return self._resolve_audio_path(
+                    filename=filename,
+                    input_ext=info.get("ext", ""),
+                    output_format=format,
+                )
+            except Exception as exc:  # pragma: no cover - passthrough from yt-dlp
+                errors.append(str(exc))
+
+        last_error = errors[-1] if errors else "unknown yt-dlp failure"
+        raise RuntimeError(
+            "Failed to download YouTube audio after multiple strategies. "
+            "Update yt-dlp (`yt-dlp -U`) and retry. "
+            f"Last error: {last_error}"
+        )
 
     def get_video_info(self, url: str) -> dict:
         """Get video information without downloading.
@@ -99,6 +99,11 @@ def get_video_info(self, url: str) -> dict:
             "quiet": True,
             "no_warnings": True,
             "extract_flat": False,
+            "extractor_args": {
+                "youtube": {
+                    "player_client": ["android", "ios", "web"],
+                }
+            },
         }
 
         with yt_dlp.YoutubeDL(ydl_opts) as ydl:
@@ -115,7 +120,67 @@ def list_formats(self, url: str) -> None:
             "listformats": True,
             "quiet": True,
             "no_warnings": True,
+            "extractor_args": {
+                "youtube": {
+                    "player_client": ["android", "ios", "web"],
+                }
+            },
         }
 
         with yt_dlp.YoutubeDL(ydl_opts) as ydl:
             ydl.extract_info(url, download=False)
+
+    def _download_strategies(self, quality: str) -> List[Dict[str, Any]]:
+        """Return prioritized yt-dlp download strategies for flaky YouTube clients."""
+        return [
+            {
+                "format": f"{quality}/bestaudio[ext=m4a]/bestaudio/best",
+                "extractor_args": {
+                    "youtube": {
+                        "player_client": ["android", "ios", "web"],
+                    }
+                },
+            },
+            {
+                "format": "bestaudio[ext=m4a]/bestaudio/best",
+                "extractor_args": {
+                    "youtube": {
+                        "player_client": ["android", "ios", "web"],
+                    }
+                },
+            },
+            {
+                "format": "140/bestaudio[ext=m4a]/bestaudio/best",
+                "extractor_args": {
+                    "youtube": {
+                        "player_client": ["android", "web"],
+                    }
+                },
+            },
+            {
+                "format": "bestaudio/best",
+            },
+        ]
+
+    def _resolve_audio_path(
+        self,
+        filename: str,
+        input_ext: str,
+        output_format: str,
+    ) -> Path:
+        """Resolve audio output path after post-processing."""
+        if input_ext != output_format:
+            audio_filename = filename.rsplit(".", 1)[0] + f".{output_format}"
+        else:
+            audio_filename = filename
+
+        audio_path = Path(audio_filename)
+        if not audio_path.exists():
+            expected_path = Path(filename.rsplit(".", 1)[0] + f".{output_format}")
+            if expected_path.exists():
+                audio_path = expected_path
+
+        if not audio_path.exists():
+            raise FileNotFoundError(f"Downloaded audio file not found: {audio_path}")
+
+        return audio_path
diff --git a/tests/test_downloader.py b/tests/test_downloader.py
@@ -0,0 +1,45 @@
+"""Tests for YouTube downloader fallback behavior."""
+
+from pathlib import Path
+from types import SimpleNamespace
+
+from transcriber import downloader
+
+
+def test_download_audio_retries_after_first_attempt_fails(monkeypatch, tmp_path):
+    """Downloader should retry with fallback options when first attempt fails."""
+    attempts = []
+    expected_output = tmp_path / "sample.mp3"
+
+    class FakeYoutubeDL:
+        def __init__(self, opts):
+            self.opts = opts
+            attempts.append(opts)
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, exc_type, exc, tb):
+            return False
+
+        def extract_info(self, url, download=True):  # noqa: ARG002
+            if len(attempts) == 1:
+                raise RuntimeError("HTTP Error 403: Forbidden")
+            expected_output.write_bytes(b"audio")
+            return {"ext": "webm", "title": "sample"}
+
+        def prepare_filename(self, info):  # noqa: ARG002
+            return str(tmp_path / "sample.webm")
+
+    monkeypatch.setattr(
+        downloader,
+        "yt_dlp",
+        SimpleNamespace(YoutubeDL=FakeYoutubeDL),
+    )
+    monkeypatch.setattr(downloader, "YT_DLP_AVAILABLE", True)
+
+    dl = downloader.YouTubeDownloader(output_dir=str(tmp_path))
+    result = dl.download_audio("https://youtu.be/test-id", quality="bestaudio", format="mp3")
+
+    assert result == Path(expected_output)
+    assert len(attempts) == 2
diff --git a/whisper-transcribe-with-download.sh b/whisper-transcribe-with-download.sh
@@ -123,10 +123,102 @@ cleanup_old_audio() {
     print_color "$BLUE" "🧹 Cleaning up audio files older than 7 days..." >&2
     find "$AUDIO_DOWNLOAD_DIR" -name "*_live_recording.wav" -type f -mtime +7 -delete 2>/dev/null
     find "$AUDIO_DOWNLOAD_DIR" -name "*_audio.mp3" -type f -mtime +7 -delete 2>/dev/null
-    find "$AUDIO_DOWNLOAD_DIR" -name "*_video.mp4" -type f -mtime +7 -delete 2>/dev/null
+    find "$AUDIO_DOWNLOAD_DIR" -name "*_video.*" -type f -mtime +7 -delete 2>/dev/null
     print_color "$GREEN" "✅ Cleanup completed" >&2
 }
 
+download_youtube_audio_with_fallback() {
+    local youtube_url=$1
+    local output_template=$2
+
+    local -a formats=(
+        "bestaudio[ext=m4a]/bestaudio/best"
+        "140/bestaudio[ext=m4a]/bestaudio/best"
+        "bestaudio/best"
+    )
+    local -a clients=(
+        "youtube:player_client=android,ios,web"
+        "youtube:player_client=android,web"
+        ""
+    )
+
+    for i in "${!formats[@]}"; do
+        local format_selector="${formats[$i]}"
+        local extractor_args="${clients[$i]}"
+
+        print_color "$YELLOW" "Trying download profile $((i+1))/${#formats[@]}..." >&2
+        local -a cmd=(
+            yt-dlp
+            --no-playlist
+            --retries 10
+            --fragment-retries 10
+            --force-ipv4
+            --concurrent-fragments 1
+            -f "$format_selector"
+            --extract-audio
+            --audio-format mp3
+            --audio-quality 0
+            -o "$output_template"
+        )
+
+        if [ -n "$extractor_args" ]; then
+            cmd+=(--extractor-args "$extractor_args")
+        fi
+        cmd+=("$youtube_url")
+
+        "${cmd[@]}" >&2
+        if [ $? -eq 0 ]; then
+            return 0
+        fi
+    done
+
+    return 1
+}
+
+download_youtube_video_with_fallback() {
+    local youtube_url=$1
+    local output_template=$2
+
+    local -a formats=(
+        "bv*[height<=720]+ba[ext=m4a]/b[height<=720]/best[height<=720]"
+        "best[height<=720]/best"
+    )
+    local -a clients=(
+        "youtube:player_client=android,ios,web"
+        ""
+    )
+
+    for i in "${!formats[@]}"; do
+        local format_selector="${formats[$i]}"
+        local extractor_args="${clients[$i]}"
+
+        print_color "$YELLOW" "Trying video profile $((i+1))/${#formats[@]}..." >&2
+        local -a cmd=(
+            yt-dlp
+            --no-playlist
+            --retries 10
+            --fragment-retries 10
+            --force-ipv4
+            --concurrent-fragments 1
+            --merge-output-format mp4
+            -f "$format_selector"
+            -o "$output_template"
+        )
+
+        if [ -n "$extractor_args" ]; then
+            cmd+=(--extractor-args "$extractor_args")
+        fi
+        cmd+=("$youtube_url")
+
+        "${cmd[@]}" >&2
+        if [ $? -eq 0 ]; then
+            return 0
+        fi
+    done
+
+    return 1
+}
+
 # Function for ORIGINAL live transcription
 original_live_transcription() {
     local model_file=$1
@@ -296,7 +388,7 @@ case $choice in
     2) # YouTube Video + Transcript
         read -p "Enter YouTube URL: " youtube_url
         print_color "$BLUE" "\n📥 Downloading YouTube audio..."
-        yt-dlp -f "bestaudio/best" --extract-audio --audio-format mp3 --audio-quality 0 -o "$AUDIO_DOWNLOAD_DIR/${TIMESTAMP}_audio.%(ext)s" "$youtube_url" >&2
+        download_youtube_audio_with_fallback "$youtube_url" "$AUDIO_DOWNLOAD_DIR/${TIMESTAMP}_audio.%(ext)s"
         if [ $? -eq 0 ]; then
             audio_file=$(find "$AUDIO_DOWNLOAD_DIR" -name "${TIMESTAMP}_audio.mp3" -type f -print -quit)
             if [ -n "$audio_file" ]; then
@@ -310,12 +402,15 @@ case $choice in
                     print_color "$GREEN" "✅ Transcript saved: $TRANSCRIPT_DIR/${TIMESTAMP}_youtube.txt"
                 fi
             fi
+        else
+            print_color "$RED" "❌ YouTube audio download failed after all fallback attempts."
+            print_color "$YELLOW" "Tip: update yt-dlp with: yt-dlp -U" >&2
         fi
         ;;
     3) # YouTube Video Download Only
         read -p "Enter YouTube URL: " youtube_url
         print_color "$BLUE" "\n📥 Downloading YouTube video..."
-        yt-dlp -f "best[height<=720]" -o "$AUDIO_DOWNLOAD_DIR/${TIMESTAMP}_video.%(ext)s" "$youtube_url" >&2
+        download_youtube_video_with_fallback "$youtube_url" "$AUDIO_DOWNLOAD_DIR/${TIMESTAMP}_video.%(ext)s"
         if [ $? -eq 0 ]; then
             video_file=$(find "$AUDIO_DOWNLOAD_DIR" -name "${TIMESTAMP}_video.*" -type f -print -quit)
             if [ -n "$video_file" ]; then
@@ -326,7 +421,8 @@ case $choice in
                 print_color "$RED" "❌ Video download failed!"
             fi
         else
-            print_color "$RED" "❌ Video download failed!"
+            print_color "$RED" "❌ Video download failed after all fallback attempts!"
+            print_color "$YELLOW" "Tip: update yt-dlp with: yt-dlp -U" >&2
         fi
         ;;
     4) # Zoom Recording
