Merge pull request #483 from dahlia/feature/media-proxy

Media proxy

Author: dahlia

AGENTS.md

@@ -426,6 +426,8 @@ STORAGE_URL_BASE=https://your-bucket.s3.amazonaws.com
 | `REMOTE_REPLIES_SCRAPE_INTERVAL_SECONDS` | 5       | Delay between scrape requests per origin |
 | `REMOTE_REPLIES_SCRAPE_BACKOFF_SECONDS`  | 300     | Backoff for 429 without `Retry-After`    |
 | `REMOTE_REPLIES_SCRAPE_COOLDOWN_SECONDS` | 300     | Completed scrape deduplication window    |
+| `MEDIA_PROXY`                            | off     | Remote media proxy: `off`, `proxy`, `cache` (booleans accepted: `true`→`proxy`, `false`→`off`) |
+| `REMOTE_MEDIA_THUMBNAILS`                | on      | Generate local sharp thumbnails for remote attachments (boolean) |
 
 
 Adding new environment variables

CHANGES.md

@@ -6,6 +6,47 @@ Version 0.9.0
 
 To be released.
 
+ -  Added a media proxy that re-serves remote avatars, headers, post
+    attachments, custom emojis, and preview-card images from Hollo's own
+    origin.  This sidesteps CORS configurations on remote object stores
+    and prevents the visitor's browser from talking directly to the
+    source server.  Controlled by a new `MEDIA_PROXY` environment
+    variable with three levels:  [[#481]]
+
+     -  `off` (default): the Mastodon API and web UI hand the original
+        remote URL to clients, matching the historical behaviour.
+     -  `proxy`: every remote media URL is rewritten to a signed
+        `/proxy/<sig>/<b64url>` path served by Hollo itself.  The proxy
+        runs SSRF checks on the upstream URL and on every redirect
+        target, allows only image/video/audio Content-Types (image/svg+xml
+        is explicitly blocked to avoid same-origin XSS), caps the body at
+        32 MiB, and serves the response with
+        `Cache-Control: public, max-age=2592000, immutable` and
+        `X-Content-Type-Options: nosniff`.  No on-disk cache.
+     -  `cache`: same URL rewriting, but the streamed body is persisted
+        to the configured storage backend as `proxy/<sha256>.bin`, with
+        a content-type sidecar alongside it at `proxy/<sha256>.json`.
+        Subsequent requests skip the upstream fetch.  The admin
+        dashboard at */thumbnail_cleanup* can purge the cache on demand.
+
+    `MEDIA_PROXY` also accepts the Boolean synonyms `true`/`on`/`1`
+    (as aliases for `proxy`) and `false`/`off`/`0` (as aliases for
+    `off`).  Disk caching is opt-in only via the explicit `cache`
+    value.
+
+    Outbound federation is unaffected: Hollo still publishes the
+    original remote URLs in ActivityPub `icon`, `image`, `attachment`,
+    and emoji `Tag` references.
+
+ -  Added a `REMOTE_MEDIA_THUMBNAILS` environment variable that controls
+    whether Hollo downloads incoming remote attachments to generate a
+    local WebP thumbnail.  Set to `off` to skip the upstream fetch and
+    Sharp pipeline entirely, storing the remote URL itself as the
+    thumbnail URL—useful in combination with `MEDIA_PROXY=proxy` or
+    `cache` to free up the disk space the local thumbnails would
+    otherwise occupy.  Defaults to `on` (the historical behavior).
+    [[#481]]
+
  -  Added [FEP-044f] quote authorization and policy support on top of the
     Mastodon-compatible quote APIs.  [[#457], [#459], [#460]]
 
@@ -156,6 +197,7 @@ To be released.
 [#466]: https://github.com/fedify-dev/hollo/pull/466
 [#467]: https://github.com/fedify-dev/hollo/pull/467
 [#479]: https://github.com/fedify-dev/hollo/issues/479
+[#481]: https://github.com/fedify-dev/hollo/issues/481
 [#482]: https://github.com/fedify-dev/hollo/pull/482
 
 

compose-fs.yaml

@@ -11,6 +11,8 @@ services:
       DRIVE_DISK: fs
       STORAGE_URL_BASE: http://localhost:3000/assets/
       FS_STORAGE_PATH: /var/lib/hollo
+      # MEDIA_PROXY: off                # or "proxy" / "cache" — see docs/install/env
+      # REMOTE_MEDIA_THUMBNAILS: "on"   # set to "off" to skip local thumbnails
     depends_on:
       - postgres
     volumes:

compose.yaml

@@ -16,6 +16,8 @@ services:
       S3_FORCE_PATH_STYLE: "true"
       AWS_ACCESS_KEY_ID: minioadmin
       AWS_SECRET_ACCESS_KEY: minioadmin
+      # MEDIA_PROXY: off                # or "proxy" / "cache" — see docs/install/env
+      # REMOTE_MEDIA_THUMBNAILS: "on"   # set to "off" to skip local thumbnails
     depends_on:
       - postgres
       - minio

docs/src/content/docs/install/env.mdx

@@ -143,6 +143,51 @@ duplicate jobs for the same replies collection.
 
 `300` by default.
 
+### `MEDIA_PROXY` <Badge text="Optional" />
+
+Controls how Hollo serves media that lives on remote servers (avatars,
+headers, attachments, custom emojis, preview-card images).  Valid values
+are:
+
+ -  `off` (default): the Mastodon API and web UI hand the original remote
+    URL to clients, matching the historical behaviour.
+ -  `proxy`: every remote media URL is rewritten to a signed
+    `/proxy/<sig>/<b64url>` path served by Hollo itself.  Hollo streams
+    the upstream bytes through on each request and does not write them
+    to disk.  Clients see only the Hollo origin, sidestepping remote
+    CORS configuration and leaks of the visitor's IP address.
+ -  `cache`: same URL rewriting as `proxy`, but the streamed body is
+    persisted to the configured storage backend as `proxy/<sha256>.bin`
+    alongside a content-type sidecar at `proxy/<sha256>.json`.
+    Subsequent requests skip the upstream fetch.  The admin dashboard
+    at */thumbnail_cleanup* can purge the cache on demand.
+
+The boolean synonyms `true` / `on` / `1` are accepted as aliases for
+`proxy`, and `false` / `off` / `0` as aliases for `off`.  Disk caching
+must be requested explicitly with `cache`.
+
+In `proxy` and `cache` modes, Hollo refuses non-HTTP(S) schemes, runs
+SSRF checks on each upstream URL and every redirect target, enforces a
+32 MiB body cap, and never proxies image/svg+xml — SVG could carry
+inline scripts that execute under the Hollo origin.
+
+`off` by default.
+
+### `REMOTE_MEDIA_THUMBNAILS` <Badge text="Optional" />
+
+Controls whether Hollo downloads remote media attachments to generate a
+local WebP thumbnail when it ingests a post.  Accepts `on` / `true` /
+`1` (the historical behaviour) or `off` / `false` / `0`.
+
+When set to `off`, Hollo skips the upstream fetch and Sharp pipeline
+entirely for incoming attachments, storing the remote URL itself as the
+thumbnail URL.  Combined with `MEDIA_PROXY=proxy` or `cache`, clients
+still see a same-origin URL at render time; with `MEDIA_PROXY=off`,
+they receive the upstream URL directly.  This frees up significant
+disk space on instances that ingest many media-heavy posts.
+
+`on` by default.
+
 ### `REMOTE_ACTOR_STALENESS_DAYS` <Badge text="Optional" />
 
 The number of days after which a remote actor's cached data is considered stale.

docs/src/content/docs/ja/install/env.mdx

@@ -141,6 +141,55 @@ HolloがL7ロードバランサーの後ろにある場合（通常はそうす
 
 デフォルトは`300`です。
 
+### `MEDIA_PROXY` <Badge text="オプション" />
+
+リモートサーバー上のメディア（アバター、ヘッダー、添付メディア、
+カスタム絵文字、プレビューカードの画像）をHolloがクライアントに対して
+どう配信するかを制御します。指定可能な値：
+
+ -  `off`（デフォルト）：MastodonクライアントAPIとウェブUIは
+    リモートのURLをそのままクライアントに渡します。これまでと同じ
+    動作です。
+ -  `proxy`：すべてのリモートメディアURLを署名付きの
+    `/proxy/<sig>/<b64url>` パスに書き換えます。Holloはリクエストごとに
+    アップストリームのバイト列をストリーミングして応答し、ディスクには
+    書き込みません。クライアントからはHolloのオリジンしか見えないため、
+    リモートサーバーのCORS設定の影響を受けず、訪問者のIPアドレスも
+    外部に漏れません。
+ -  `cache`：URLの書き換えは`proxy`と同じですが、ストリーミングした
+    本文を設定済みのストレージバックエンドの`proxy/<sha256>.bin`に
+    保存し、Content-Type情報を持つサイドカーを`proxy/<sha256>.json`に
+    一緒に保存します。以降のリクエストはアップストリームを再取得
+    しません。管理ダッシュボードの */thumbnail_cleanup* から必要に
+    応じてキャッシュを消去できます。
+
+真偽値の同義語として`true` / `on` / `1`は`proxy`の別名、
+`false` / `off` / `0`は`off`の別名として受け付けます。ディスク
+キャッシュは`cache`で明示的に有効化する必要があります。
+
+`proxy`と`cache`モードでは、HolloはHTTP(S)以外のスキームをプロキシ
+しません。アップストリームのURLとすべてのリダイレクト先に対してSSRF
+チェックを行い、本文は32 MiBに制限し、image/svg+xmlはHolloのオリジン
+上で実行され得るインラインスクリプトを含むため一切プロキシしません。
+
+デフォルトは`off`です。
+
+### `REMOTE_MEDIA_THUMBNAILS` <Badge text="オプション" />
+
+投稿を取り込む際に、Holloがリモートの添付メディアをダウンロードして
+ローカルのWebPサムネイルを生成するかを制御します。値は`on` / `true` /
+`1`（これまでの動作）または`off` / `false` / `0`です。
+
+`off`に設定すると、Holloは受信した添付に対してアップストリームの取得
+とSharpパイプラインを完全にスキップし、リモートURLをそのまま
+サムネイルURLとして保存します。`MEDIA_PROXY=proxy`または`cache`と
+組み合わせれば、クライアントはレンダリング時に依然として同一オリジンの
+URLを受け取り、`MEDIA_PROXY=off`の場合はアップストリームのURLを直接
+受け取ります。多数のメディア投稿を取り込むインスタンスでは、大量の
+ディスク容量を節約できます。
+
+デフォルトは`on`です。
+
 ### `REMOTE_ACTOR_STALENESS_DAYS` <Badge text="オプション" />
 
 リモートアクターのキャッシュされたデータが古いと見なされるまでの日数。

docs/src/content/docs/ko/install/env.mdx

@@ -139,6 +139,51 @@ Hollo가 L7 로드 밸런서 뒤에 위치할 경우 (일반적으로 그래야
 
 기본값은 `300`입니다.
 
+### `MEDIA_PROXY` <Badge text="선택" />
+
+다른 서버에 있는 미디어(아바타, 헤더, 첨부 미디어, 커스텀 이모지,
+미리보기 카드 이미지)를 Hollo가 어떻게 클라이언트에게 전달할지
+결정합니다.  지정 가능한 값:
+
+ -  `off` (기본값): Mastodon API와 웹 UI가 원격 URL을 그대로 전달합니다.
+    이전과 동일한 동작입니다.
+ -  `proxy`: 모든 원격 미디어 URL을 서명된 `/proxy/<sig>/<b64url>`
+    경로로 재작성합니다.  Hollo가 매 요청마다 원본을 스트리밍해 응답하며
+    디스크에는 저장하지 않습니다.  클라이언트는 Hollo 도메인만 보게 되어
+    원격 서버의 CORS 설정에 영향을 받지 않고, 방문자 IP도 외부로
+    노출되지 않습니다.
+ -  `cache`: `proxy`와 동일한 URL 재작성에 더해, 스트리밍한 본문을 설정된
+    저장소 백엔드의 `proxy/<sha256>.bin`에 저장하고, 콘텐츠 타입 정보를
+    담은 사이드카 파일을 `proxy/<sha256>.json`에 함께 저장합니다.
+    이후 요청은 원본을 다시 가져오지 않습니다.  */thumbnail_cleanup*
+    관리자 페이지에서 필요할 때 캐시를 비울 수 있습니다.
+
+불리언 동의어로 `true` / `on` / `1`은 `proxy`의 별칭으로,
+`false` / `off` / `0`은 `off`의 별칭으로 받아들입니다.  디스크 캐싱은
+반드시 `cache`로 명시적으로 요청해야 합니다.
+
+`proxy`와 `cache` 모드에서 Hollo는 HTTP(S)가 아닌 스킴은 프록시하지
+않고, 원본 URL과 모든 리다이렉트 대상에 대해 SSRF 검사를 수행하며,
+본문 크기를 32 MiB로 제한하고, image/svg+xml은 Hollo 도메인에서
+실행되는 인라인 스크립트를 포함할 수 있어 절대 프록시하지 않습니다.
+
+기본값은 `off`입니다.
+
+### `REMOTE_MEDIA_THUMBNAILS` <Badge text="선택" />
+
+게시물을 수신할 때 Hollo가 원격 첨부 미디어를 내려받아 로컬 WebP
+썸네일을 생성할지 결정합니다.  값은 `on` / `true` / `1`(이전과 동일한
+동작) 또는 `off` / `false` / `0`입니다.
+
+`off`로 설정하면 Hollo는 들어오는 첨부에 대해 원본 다운로드와 Sharp
+파이프라인을 건너뛰고, 원격 URL을 그대로 썸네일 URL로 저장합니다.
+`MEDIA_PROXY=proxy` 또는 `cache`와 함께 쓰면 클라이언트는 렌더 시
+여전히 같은 출처의 URL을 보게 되며, `MEDIA_PROXY=off`일 때는 원본
+URL을 그대로 받습니다.  미디어가 많은 게시물을 자주 받는 인스턴스에서
+디스크 공간을 크게 절약할 수 있습니다.
+
+기본값은 `on`입니다.
+
 ### `REMOTE_ACTOR_STALENESS_DAYS` <Badge text="선택" />
 
 원격 액터의 캐시된 데이터가 오래된 것으로 간주되기까지의 일수.

docs/src/content/docs/zh-cn/install/env.mdx

@@ -129,6 +129,46 @@ openssl rand -hex 32
 
 默认为`300`。
 
+### `MEDIA_PROXY` <Badge text="可选" />
+
+控制 Hollo 如何向客户端提供位于远程服务器的媒体（头像、横幅图、附件、
+自定义表情、链接预览卡片图片）。可用值：
+
+ -  `off`（默认）：Mastodon API 和 Web UI 将远程 URL 原样交给客户端，
+    与历史行为一致。
+ -  `proxy`：将所有远程媒体 URL 改写为带签名的
+    `/proxy/<sig>/<b64url>` 路径，由 Hollo 自己提供。Hollo 在每次请求时
+    将上游字节流式转发，不写入磁盘。客户端只看到 Hollo 自身的源,
+    避免远程 CORS 配置的影响，也避免访问者的 IP 被泄露。
+ -  `cache`：URL 改写与 `proxy` 相同，但会把流式获取的响应主体保存到
+    所配置存储后端的 `proxy/<sha256>.bin`,并把记录内容类型的旁路文件
+    保存到 `proxy/<sha256>.json`。后续请求会跳过上游请求。管理面板的
+    */thumbnail_cleanup* 页面可以按需清空缓存。
+
+布尔同义值：`true` / `on` / `1` 作为 `proxy` 的别名，
+`false` / `off` / `0` 作为 `off` 的别名。磁盘缓存必须用 `cache` 显式
+开启。
+
+在 `proxy` 和 `cache` 模式下，Hollo 会拒绝代理非 HTTP(S) 协议,对上游
+URL 和每一次重定向目标执行 SSRF 检查,将响应主体限制为 32 MiB,并且
+绝不代理 image/svg+xml——SVG 可能包含会在 Hollo 自身源上执行的内联
+脚本。
+
+默认为`off`。
+
+### `REMOTE_MEDIA_THUMBNAILS` <Badge text="可选" />
+
+控制 Hollo 接收帖子时是否下载远程附件媒体以生成本地 WebP 缩略图。
+可用值为 `on` / `true` / `1`（历史行为）或 `off` / `false` / `0`。
+
+设置为 `off` 时，Hollo 会对收到的附件完全跳过上游下载和 Sharp 流程，
+直接将远程 URL 作为缩略图 URL 存储。配合 `MEDIA_PROXY=proxy` 或
+`cache` 使用时，客户端在渲染时仍然得到同源 URL；若 `MEDIA_PROXY=off`，
+客户端会直接收到上游 URL。在接收大量媒体型帖子的实例上可以显著
+节省磁盘空间。
+
+默认为`on`。
+
 ### `REMOTE_ACTOR_STALENESS_DAYS` <Badge text="可选" />
 
 远程用户的缓存数据被视为过期的天数。

src/api/v1/index.ts

@@ -8,6 +8,7 @@ import { db } from "../../db";
 import { serializeAccount } from "../../entities/account";
 import { getPostRelations, serializePost } from "../../entities/status";
 import { serializeTag } from "../../entities/tag";
+import { proxyUrl } from "../../media-proxy";
 import {
   scopeRequired,
   tokenRequired,
@@ -73,14 +74,21 @@ app.get(
 
 app.get("/custom_emojis", async (c) => {
   const emojis = await db.query.customEmojis.findMany();
+  const baseUrl = c.req.url;
   return c.json(
-    emojis.map((emoji) => ({
-      shortcode: emoji.shortcode,
-      url: emoji.url,
-      static_url: emoji.url,
-      visible_in_picker: true,
-      category: emoji.category,
-    })),
+    emojis.flatMap((emoji) => {
+      const url = proxyUrl(emoji.url, baseUrl);
+      if (url == null) return [];
+      return [
+        {
+          shortcode: emoji.shortcode,
+          url,
+          static_url: url,
+          visible_in_picker: true,
+          category: emoji.category,
+        },
+      ];
+    }),
   );
 });
 

src/api/v1/media.ts

@@ -75,7 +75,7 @@ export async function postMedia(
   if (result.length < 1) {
     return c.json({ error: "Failed to insert media" }, 500);
   }
-  return c.json(serializeMedium(result[0]));
+  return c.json(serializeMedium(result[0], c.req.url));
 }
 
 app.post(
@@ -93,7 +93,7 @@ app.get("/:id", async (c) => {
     where: eq(media.id, mediumId),
   });
   if (medium == null) return c.json({ error: "Not found" }, 404);
-  return c.json(serializeMedium(medium));
+  return c.json(serializeMedium(medium, c.req.url));
 });
 
 app.put("/:id", tokenRequired, scopeRequired(["write:media"]), async (c) => {
@@ -116,7 +116,7 @@ app.put("/:id", tokenRequired, scopeRequired(["write:media"]), async (c) => {
     .where(eq(media.id, mediumId))
     .returning();
   if (result.length < 1) return c.json({ error: "Not found" }, 404);
-  return c.json(serializeMedium(result[0]));
+  return c.json(serializeMedium(result[0], c.req.url));
 });
 
 export default app;