diff --git a/CHANGELOG.md b/CHANGELOG.md index 560f5409..333ad6f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ - Plugin Pages 用户、Feed、订阅删除流程新增「同时清理推送历史」选项,默认保留历史审计数据。 - Plugin Pages 订阅列表新增按 Feed URL 精确筛选入口,推送历史跳转订阅不再依赖可能复用的 `sub_id`。 - 新增跨平台测试脚本 `tests/run_tests.sh`,方便 macOS/Linux 本地执行分类测试。 +- 新增媒体反代:`media.image_relay_base_url` 与 `media.media_relay_base_url`,图片优先走图片反代、非图片走通用媒体反代,下载时先反代再回源;缓存键与失败展示链接始终保持原始 URL,支持 `https://wsrv.nl/` 与 `https://wsrv.nl/?url=` 两种形式。 +- 新增无声视频转 GIF 的体积上限压缩:逐步降低分辨率与帧率,将 GIF 压缩到平台允许的大小内。 ### Changed diff --git a/README.md b/README.md index 107cc776..760e54a7 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ ![Python Version](https://img.shields.io/badge/Python-3.10%2B-blue) ![AstrBot](https://img.shields.io/badge/AstrBot-%E2%89%A54.24.0-green) ![Platform](https://img.shields.io/badge/Platform-Windows%20%7C%20Linux%20%7C%20MacOS-lightgrey) +[![Last Commit](https://img.shields.io/github/last-commit/FlanChanXwO/astrbot_plugin_rsshub)](https://github.com/FlanChanXwO/astrbot_plugin_rsshub/commits/master) @@ -50,6 +51,10 @@ git clone https://github.com/FlanChanXwO/astrbot_plugin_rsshub.git - 项目与架构:[`docs/project/README.md`](./docs/project/README.md) - 开发与贡献:[`docs/dev/README.md`](./docs/dev/README.md) +## 命令一览 + +![命令一览](https://raw.githubusercontent.com/FlanChanXwO/astrbot_plugin_rsshub/master/assets/help/rsshelp_light.png) + ## 协议 本项目基于 [AGPL](LICENSE) 协议开源。 diff --git a/assets/help/rsshelp_dark.png b/assets/help/rsshelp_dark.png index fc471aa2..304f9b9e 100644 Binary files a/assets/help/rsshelp_dark.png and b/assets/help/rsshelp_dark.png differ diff --git a/assets/help/rsshelp_light.png b/assets/help/rsshelp_light.png index 02573708..e6689db3 100644 Binary files a/assets/help/rsshelp_light.png and b/assets/help/rsshelp_light.png differ diff --git a/docs/usage/commands.md b/docs/usage/commands.md index b1a5d837..56829219 100644 --- a/docs/usage/commands.md +++ b/docs/usage/commands.md @@ -66,10 +66,10 @@ | --- | --- | --- | | `/rsshelp` | `/RSS帮助`, `/rss帮助` | 查看帮助图片。 | | `/sub_test <目标>` | `/测试订阅` | 管理员测试推送;目标可以是订阅 ID 或 RSS URL,固定推送最新 1 条。 | -| `/rsshub_kb_init` | - | 管理员初始化 RSSHub Routes 知识库。 | -| `/rsshub_kb_sync` | - | 管理员启动 RSSHub Routes 知识库同步任务。 | -| `/rsshub_kb_status` | - | 查看 RSSHub Routes 知识库状态。 | -| `/rsshub_kb_task` | - | 查看最近一次 Routes KB 同步任务进度。 | +| `/rsshub_kb_init` | `/rss知识库初始化`, `/RSS知识库初始化` | 管理员初始化 RSSHub Routes 知识库。 | +| `/rsshub_kb_sync` | `/rss知识库同步`, `/RSS知识库同步` | 管理员启动 RSSHub Routes 知识库同步任务。 | +| `/rsshub_kb_status` | `/rss知识库同步状态`, `/RSS知识库同步状态` | 查看 RSSHub Routes 知识库状态。 | +| `/rsshub_kb_task` | `/rss知识库近期同步任务`, `/RSS知识库近期同步任务` | 查看最近一次 Routes KB 同步任务进度。 | `rsshelp` 使用仓库内预生成图片:白天发送 `assets/help/rsshelp_light.png`,夜间发送 `assets/help/rsshelp_dark.png`。命令或帮助样式变化后可手动运行: diff --git a/scripts/template/rsshelp_template.html b/scripts/template/rsshelp_template.html index 09538636..ff063ec0 100644 --- a/scripts/template/rsshelp_template.html +++ b/scripts/template/rsshelp_template.html @@ -16,7 +16,7 @@ {{ rsslogo_svg | safe }}
RSSHub for AstrBot 命令帮助
-
预生成静态帮助图 · 命令说明来自插件入口定义
+
Everything is RSSable
diff --git a/src/infrastructure/media/media_downloader.py b/src/infrastructure/media/media_downloader.py index f7960a64..bba3bd72 100644 --- a/src/infrastructure/media/media_downloader.py +++ b/src/infrastructure/media/media_downloader.py @@ -11,7 +11,7 @@ import tempfile import time from pathlib import Path -from urllib.parse import parse_qs, unquote, urlparse +from urllib.parse import parse_qs, quote, unquote, urlparse import aiohttp @@ -54,6 +54,40 @@ ) +def _build_relay_url(base: str | None, original_url: str) -> str: + """把原始媒体 URL 包装成反代地址;base 为空(含 None)时原样返回。 + + 支持 https://wsrv.nl/ 与 https://wsrv.nl/?url= 两种形式:以 '=' 结尾视为 + 前缀直接追加编码后的原始 URL;含 '?' 时用 '&url=' 追加;否则归一为 + '/?url='。 + """ + base = (base or "").strip() + if not base: + return original_url + encoded = quote(original_url, safe="") + if base.endswith("="): + return base + encoded + if "?" in base: + return base + "&url=" + encoded + return base.rstrip("/") + "/?url=" + encoded + + +def _select_relay_base( + media_type: str | None, + image_relay_base_url: str | None, + media_relay_base_url: str | None, +) -> str: + """图片优先走图片反代;图片反代未配置或非图片走通用媒体反代;都未配置返回空串。 + + 两个 base 入参允许为 None(视作未配置)。 + """ + img = (image_relay_base_url or "").strip() + media = (media_relay_base_url or "").strip() + if media_type == "image": + return img or media + return media + + class MediaDownloader: """媒体下载器,支持缓存管理和格式转换。""" @@ -616,6 +650,8 @@ async def get_or_download( try_convert_gif: bool = False, gif_transcode_timeout: int = 60, m3u8_timeout: int = 120, + image_relay_base_url: str | None = "", + media_relay_base_url: str | None = "", ) -> Path: """下载媒体到缓存,支持 GIF 自动转换 @@ -627,6 +663,8 @@ async def get_or_download( try_convert_gif: 是否尝试将无声视频转为GIF gif_transcode_timeout: GIF转码超时 m3u8_timeout: m3u8下载超时 + image_relay_base_url: 图片反代 base URL,配置后图片走该反代抓取 + media_relay_base_url: 通用媒体反代 base URL,配置后非图片媒体走该反代抓取 Returns: 缓存文件路径 @@ -663,11 +701,31 @@ async def get_or_download( bool(proxy), try_convert_gif, ) - tmp_path = await self.download_to_temp( - url=url, - timeout_seconds=timeout_seconds, - proxy=proxy, + relay_base = _select_relay_base( + media_hint.media_type or media_type, + image_relay_base_url, + media_relay_base_url, ) + fetch_url = _build_relay_url(relay_base, url) + if fetch_url != url: + try: + tmp_path = await self.download_to_temp( + url=fetch_url, timeout_seconds=timeout_seconds, proxy=proxy + ) + except Exception as ex: + logger.warning( + "媒体反代下载失败,回源直连: relay=%s, url=%s, err=%s", + fetch_url, + url, + ex, + ) + tmp_path = await self.download_to_temp( + url=url, timeout_seconds=timeout_seconds, proxy=proxy + ) + else: + tmp_path = await self.download_to_temp( + url=url, timeout_seconds=timeout_seconds, proxy=proxy + ) logger.debug( "Media cache download complete: url=%s, tmp=%s, tmp_exists=%s", url, @@ -769,8 +827,15 @@ async def get_or_download_prepared( try_convert_gif: bool = False, gif_transcode_timeout: int = 60, m3u8_timeout: int = 120, + image_relay_base_url: str | None = "", + media_relay_base_url: str | None = "", ): - """下载媒体并返回 PreparedMedia,保留原始视频与 GIF 变体。""" + """下载媒体并返回 PreparedMedia,保留原始视频与 GIF 变体。 + + Args: + image_relay_base_url: 图片反代 base URL,配置后图片走该反代抓取。 + media_relay_base_url: 通用媒体反代 base URL,配置后非图片媒体走该反代抓取。 + """ from ..messaging.senders.types import MediaVariant, PreparedMedia local_path = await self.get_or_download( @@ -781,6 +846,8 @@ async def get_or_download_prepared( try_convert_gif=False, gif_transcode_timeout=gif_transcode_timeout, m3u8_timeout=m3u8_timeout, + image_relay_base_url=image_relay_base_url, + media_relay_base_url=media_relay_base_url, ) detection = detect_media_file(local_path) effective_type = ( diff --git a/src/infrastructure/utils/ffmpeg_helper.py b/src/infrastructure/utils/ffmpeg_helper.py index 9dc00cbf..3f25bb9c 100644 --- a/src/infrastructure/utils/ffmpeg_helper.py +++ b/src/infrastructure/utils/ffmpeg_helper.py @@ -545,6 +545,159 @@ async def transcode_to_gif( return None + _GIF_COMPRESS_SCALE_FACTORS: Final = [0.75, 0.5, 0.35, 0.25] + _GIF_COMPRESS_MIN_FPS: Final = 15 + + @staticmethod + async def transcode_to_gif_under_limit( + source_path: Path, + *, + max_bytes: int, + timeout_seconds: int = 60, + auto_install_ffmpeg: bool = True, + ) -> Path | None: + """将视频转码为不超过 max_bytes 的 GIF。 + + 逐步降低分辨率和帧率来压缩,直到满足大小限制。 + + Args: + source_path: 源视频文件路径 + max_bytes: GIF 最大字节数 + timeout_seconds: 单次转码超时(秒) + auto_install_ffmpeg: 是否自动安装 ffmpeg + + Returns: + 生成的 GIF 路径,或 None(失败时) + """ + ffmpeg_exe = FFmpegTool.ensure_ffmpeg_ready(auto_install=auto_install_ffmpeg) + if not ffmpeg_exe: + return None + + if not source_path.exists() or not source_path.is_file(): + return None + + try: + stat = source_path.stat() + except OSError: + return None + + cache_root = get_plugin_cache_dir("gif_compressed") + cache_root.mkdir(parents=True, exist_ok=True) + + attempts = [ + (FFmpegTool._GIF_TRANSCODE_FPS, factor) + for factor in FFmpegTool._GIF_COMPRESS_SCALE_FACTORS + ] + [ + (FFmpegTool._GIF_COMPRESS_MIN_FPS, factor) + for factor in FFmpegTool._GIF_COMPRESS_SCALE_FACTORS + ] + + for fps, scale_factor in attempts: + cache_key = ( + f"{source_path.resolve()}::" + f"{int(stat.st_mtime)}::{stat.st_size}::" + f"compressed:{max_bytes}:{fps}:{scale_factor}" + ) + digest = hashlib.sha256( + cache_key.encode("utf-8", errors="ignore") + ).hexdigest() + output_path = cache_root / f"{digest}.gif" + + if output_path.exists(): + cached_size = output_path.stat().st_size + if cached_size > 0: + if cached_size <= max_bytes: + return output_path + continue + + scale_w = f"iw*{scale_factor}" + vf_expr = ( + f"fps={fps}," + f"scale={scale_w}:-1:flags=lanczos," + "split[s0][s1];" + f"[s0]palettegen=max_colors={FFmpegTool._GIF_TRANSCODE_MAX_COLORS}" + ":stats_mode=full[p];" + f"[s1][p]paletteuse=dither={FFmpegTool._GIF_TRANSCODE_DITHER}" + ) + args = [ + ffmpeg_exe, + "-y", + "-i", + str(source_path), + "-vf", + vf_expr, + "-loop", + "0", + str(output_path), + ] + + process: asyncio.subprocess.Process | None = None + try: + process = await asyncio.create_subprocess_exec( + *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _stdout, stderr = await asyncio.wait_for( + process.communicate(), + timeout=max(10, int(timeout_seconds)), + ) + except asyncio.TimeoutError: + logger.warning( + "FFmpeg compressed GIF timeout: src=%s, scale=%s", + source_path, + scale_factor, + ) + if process is not None: + process.kill() + await process.wait() + output_path.unlink(missing_ok=True) + continue + except (OSError, ValueError) as ex: + logger.warning( + "FFmpeg compressed GIF failed: src=%s, err=%s", + source_path, + ex, + ) + continue + + if process.returncode != 0: + output_path.unlink(missing_ok=True) + continue + + if not output_path.exists(): + output_path.unlink(missing_ok=True) + continue + output_size = output_path.stat().st_size + if output_size == 0: + output_path.unlink(missing_ok=True) + continue + + if output_size <= max_bytes: + logger.debug( + "Compressed GIF success: src=%s, bytes=%s, scale=%s, fps=%s", + source_path, + output_size, + scale_factor, + fps, + ) + return output_path + + logger.debug( + "Compressed GIF still too large: src=%s, bytes=%s > %s, scale=%s", + source_path, + output_size, + max_bytes, + scale_factor, + ) + + logger.warning( + "Failed to compress GIF under limit: src=%s, max_bytes=%s", + source_path, + max_bytes, + ) + return None + @staticmethod async def download_m3u8_to_mp4( m3u8_url: str, diff --git a/tests/unit/infrastructure/test_base_sender_ffmpeg.py b/tests/unit/infrastructure/test_base_sender_ffmpeg.py index 5b5bee96..cfb96141 100644 --- a/tests/unit/infrastructure/test_base_sender_ffmpeg.py +++ b/tests/unit/infrastructure/test_base_sender_ffmpeg.py @@ -3,6 +3,10 @@ from pathlib import Path import pytest +from astrbot_plugin_rsshub.src.domain.entities.content_types import ( + LayoutFragment, + build_generated_media_url, +) from astrbot_plugin_rsshub.src.infrastructure.messaging.senders.base_sender import ( DefaultMessageSender, ) @@ -14,10 +18,6 @@ PreparedMedia, SendRequest, ) -from astrbot_plugin_rsshub.src.domain.entities.content_types import ( - LayoutFragment, - build_generated_media_url, -) class _FakeDownloader: @@ -119,6 +119,8 @@ async def test_prepare_media_passes_gif_transcode_config(monkeypatch, fake_detec "media_type": "video", "try_convert_gif": True, "gif_transcode_timeout": 77, + "image_relay_base_url": "", + "media_relay_base_url": "", } ] diff --git a/tests/unit/infrastructure/test_media_downloader.py b/tests/unit/infrastructure/test_media_downloader.py index 259eca9a..1e89dab5 100644 --- a/tests/unit/infrastructure/test_media_downloader.py +++ b/tests/unit/infrastructure/test_media_downloader.py @@ -2,6 +2,7 @@ from pathlib import Path from types import SimpleNamespace +from urllib.parse import quote import pytest from astrbot_plugin_rsshub.src.infrastructure.media.media_downloader import ( @@ -632,3 +633,229 @@ async def fake_download_m3u8_to_mp4( assert f"m3u8 download failed: {wrapped_url}" in str(exc_info.value) assert "ffmpeg returned unsuccessful result" in str(exc_info.value) assert not list(tmp_path.glob("*.mp4")) + + +# --------------------------------------------------------------------------- # +# 反代 (reverse-proxy / relay) 行为测试 # +# --------------------------------------------------------------------------- # +# 约定:网络抓取走 self.download_to_temp(url=),其中 FETCH_URL 是反代 +# 包装后的 URL;缓存键与 PreparedMedia.original_url 始终保持原始 url 不变。 + + +def _capture_download(captured: list[str], tmp_path: Path, payload: bytes): + """复用既有 download_to_temp seam:捕获 url 并落地一个临时文件。""" + + async def fake_download(self, **kwargs): + captured.append(kwargs["url"]) + path = tmp_path / f"source-{len(captured)}.gif" + path.write_bytes(payload) + return path + + return fake_download + + +@pytest.mark.asyncio +async def test_relay_image_with_base_ending_in_equals( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + # base 以 "=" 结尾 → base + quote(original) + captured: list[str] = [] + original = "https://example.com/a.gif" + base = "https://wsrv.nl/?url=" + expected_fetch = base + quote(original, safe="") + + monkeypatch.setattr( + MediaDownloader, + "download_to_temp", + _capture_download(captured, tmp_path, _VALID_GIF), + ) + + downloader = MediaDownloader(cache_dir=tmp_path) + path = await downloader.get_or_download( + url=original, + media_type="image", + image_relay_base_url=base, + ) + + assert captured == [expected_fetch] + # 缓存键 / 原始 url 仍是原始地址 + assert downloader._read_cache(original) == path + assert path.read_bytes() == _VALID_GIF + + +@pytest.mark.asyncio +async def test_relay_image_with_bare_base_appends_query( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + # base 既不以 "=" 结尾,也不含 "?" → base.rstrip("/") + "/?url=" + quote(original) + captured: list[str] = [] + original = "https://example.com/a.gif" + base = "https://wsrv.nl/" + expected_fetch = "https://wsrv.nl/?url=" + quote(original, safe="") + + monkeypatch.setattr( + MediaDownloader, + "download_to_temp", + _capture_download(captured, tmp_path, _VALID_GIF), + ) + + downloader = MediaDownloader(cache_dir=tmp_path) + path = await downloader.get_or_download( + url=original, + media_type="image", + image_relay_base_url=base, + ) + + assert captured == [expected_fetch] + assert downloader._read_cache(original) == path + + +@pytest.mark.asyncio +async def test_relay_non_image_uses_media_relay_base( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + # 非图片 (file) 只设置 media_relay_base_url → 走 media 反代。 + # bare base → base.rstrip("/") + "/?url=" + quote(original) + captured: list[str] = [] + original = "https://example.com/doc.bin" + base = "https://relay.example/" + expected_fetch = "https://relay.example/?url=" + quote(original, safe="") + + monkeypatch.setattr( + MediaDownloader, + "download_to_temp", + _capture_download(captured, tmp_path, _VALID_GIF), + ) + + downloader = MediaDownloader(cache_dir=tmp_path) + await downloader.get_or_download( + url=original, + media_type="file", + media_relay_base_url=base, + ) + + assert captured == [expected_fetch] + + +@pytest.mark.asyncio +async def test_relay_image_falls_back_to_media_relay_when_image_relay_unset( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + # 图片:image_relay 未设置但 media_relay 已设置 → 回退使用 media 反代。 + captured: list[str] = [] + original = "https://example.com/a.gif" + base = "https://relay.example/" + expected_fetch = "https://relay.example/?url=" + quote(original, safe="") + + monkeypatch.setattr( + MediaDownloader, + "download_to_temp", + _capture_download(captured, tmp_path, _VALID_GIF), + ) + + downloader = MediaDownloader(cache_dir=tmp_path) + path = await downloader.get_or_download( + url=original, + media_type="image", + image_relay_base_url="", + media_relay_base_url=base, + ) + + assert captured == [expected_fetch] + assert downloader._read_cache(original) == path + + +@pytest.mark.asyncio +async def test_relay_both_empty_fetches_original_url( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + # 两个反代 base 都为空 → 行为与之前完全一致,抓取原始 url。 + captured: list[str] = [] + original = "https://example.com/a.gif" + + monkeypatch.setattr( + MediaDownloader, + "download_to_temp", + _capture_download(captured, tmp_path, _VALID_GIF), + ) + + downloader = MediaDownloader(cache_dir=tmp_path) + path = await downloader.get_or_download( + url=original, + media_type="image", + image_relay_base_url="", + media_relay_base_url="", + ) + + assert captured == [original] + assert downloader._read_cache(original) == path + + +@pytest.mark.asyncio +async def test_relay_falls_back_to_origin_when_relay_fetch_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + # "先反代再回源":第一次(反代 url)抓取抛错 → 第二次用原始 url 重试并成功。 + captured: list[str] = [] + original = "https://example.com/a.gif" + base = "https://wsrv.nl/?url=" + relay_fetch = base + quote(original, safe="") + + async def fake_download(self, **kwargs): + captured.append(kwargs["url"]) + if kwargs["url"] == relay_fetch: + raise RuntimeError("relay status=502") + path = tmp_path / f"source-{len(captured)}.gif" + path.write_bytes(_VALID_GIF) + return path + + monkeypatch.setattr(MediaDownloader, "download_to_temp", fake_download) + + downloader = MediaDownloader(cache_dir=tmp_path) + path = await downloader.get_or_download( + url=original, + media_type="image", + image_relay_base_url=base, + ) + + assert captured == [relay_fetch, original] + assert path.read_bytes() == _VALID_GIF + # 回源时缓存键 / 原始 url 仍保持原始地址 + assert downloader._read_cache(original) == path + + +@pytest.mark.asyncio +async def test_get_or_download_prepared_passes_relay_kwargs_through( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + # get_or_download_prepared 透传反代 kwargs:抓取 url 是反代 url, + # 而 PreparedMedia.original_url 仍是原始地址。 + captured: list[str] = [] + original = "https://example.com/a.jpg" + base = "https://wsrv.nl/?url=" + expected_fetch = base + quote(original, safe="") + + async def fake_download(self, **kwargs): + captured.append(kwargs["url"]) + path = tmp_path / f"source-{len(captured)}.jpg" + path.write_bytes(_VALID_JPEG) + return path + + monkeypatch.setattr(MediaDownloader, "download_to_temp", fake_download) + + downloader = MediaDownloader(cache_dir=tmp_path) + prepared = await downloader.get_or_download_prepared( + url=original, + media_type="image", + image_relay_base_url=base, + ) + + assert captured == [expected_fetch] + assert prepared.original_url == original