优化大文件下载时的内存占用

Author: UE-DND

src/hooks/useDownload.ts

@@ -8,12 +8,18 @@
  */
 
 import { useReducer, useCallback, useRef } from "react";
-import JSZip from "jszip";
 import { saveAs } from "file-saver";
 import type { DownloadState, DownloadAction, GitHubContent } from "@/types";
 import { GitHub } from "@/services/github";
 import { logger } from "@/utils";
+import { isAbortError } from "@/utils/network/abort";
 import { requestManager } from "@/utils/request/requestManager";
+import {
+  downloadFolderAsZip,
+  prepareZipOutputSink,
+  type FolderDownloadEntry,
+  type ZipOutputSink,
+} from "@/utils/download/folderZipPipeline";
 import { getForceServerProxy } from "@/services/github/config/ProxyForceManager";
 import { useI18n } from "@/contexts/I18nContext";
 
@@ -216,7 +222,7 @@ export const useDownload = (
   const collectFiles = useCallback(
     async function collectFilesInner(
       folderPath: string,
-      fileList: { path: string; url: string }[],
+      fileList: FolderDownloadEntry[],
       basePath: string,
       signal: AbortSignal,
     ): Promise<void> {
@@ -226,6 +232,7 @@ export const useDownload = (
         const contents = await requestManager.request(
           `download-folder-${folderPath}`,
           (requestSignal) => GitHub.Content.getContents(folderPath, requestSignal),
+          { signal },
         );
 
         // 检查是否已取消 (ref可在异步期间被cancelDownload修改)
@@ -258,6 +265,7 @@ export const useDownload = (
             fileList.push({
               path: relativePath,
               url: downloadUrl,
+              size: item.size,
             });
           } else {
             // 递归处理子文件夹 (type === 'dir')
@@ -297,95 +305,62 @@ export const useDownload = (
       // 创建新的AbortController
       abortControllerRef.current = new AbortController();
       const signal = abortControllerRef.current.signal;
+      let outputSink: ZipOutputSink | null = null;
+      let zipPipelineStarted = false;
 
       try {
-        const zip = new JSZip();
+        outputSink = await prepareZipOutputSink({
+          archiveName: `${folderName}.zip`,
+          saveAsImpl: saveAs,
+        });
 
         // 递归获取文件夹内容
-        const allFiles: { path: string; url: string }[] = [];
+        const allFiles: FolderDownloadEntry[] = [];
         await collectFiles(path, allFiles, path, signal);
 
         // 检查是否已取消 (ref可在异步期间被cancelDownload修改)
         if (hasBeenCancelled()) {
+          await outputSink.abort();
           logger.info("文件夹下载已取消");
           return;
         }
 
         dispatch({ type: "SET_TOTAL_FILES", count: allFiles.length });
         logger.info(`需要下载的文件总数: ${String(allFiles.length)}`);
 
-        // 下载并添加到zip
         let processedCount = 0;
-        for (const file of allFiles) {
-          try {
-            // 检查是否已取消 (ref可在异步期间被cancelDownload修改)
-            if (hasBeenCancelled()) {
-              logger.info("文件夹下载已取消");
-              return;
-            }
-
-            const response = await fetch(file.url, { signal });
-
-            if (!response.ok) {
-              logger.error(
-                `文件 ${file.path} 下载失败:`,
-                new Error(`下载失败: ${String(response.status)}`),
-              );
-              continue;
-            }
-
-            const blob = await response.blob();
-            zip.file(file.path, blob);
-
-            processedCount++;
-            dispatch({ type: "SET_PROCESSING_FILES", count: processedCount });
+        zipPipelineStarted = true;
+        await downloadFolderAsZip({
+          files: allFiles,
+          signal,
+          archiveName: `${folderName}.zip`,
+          outputSink,
+          onFileComplete: (count, total) => {
+            processedCount = count;
+            dispatch({ type: "SET_PROCESSING_FILES", count });
             dispatch({
               type: "SET_FOLDER_PROGRESS",
-              progress: Math.round((processedCount / allFiles.length) * 100),
+              progress: total > 0 ? Math.round((count / total) * 100) : 100,
             });
-          } catch (e) {
-            // 检查是否是取消导致的错误
-            if (e instanceof Error && (e.name === "AbortError" || hasBeenCancelled())) {
-              logger.info("文件夹下载已取消");
-              return;
-            }
-            logger.error(`文件 ${file.path} 下载失败:`, e);
-          }
-
-          // 检查是否已取消 (ref可在异步期间被cancelDownload修改)
-          if (hasBeenCancelled()) {
-            logger.info("文件夹下载已取消");
-            return;
-          }
-        }
-
-        // 生成zip文件
-        const zipBlob = await zip.generateAsync(
-          {
-            type: "blob",
-            compression: "DEFLATE",
-            compressionOptions: { level: 6 },
           },
-          (metadata: { percent: number }) => {
-            // 检查是否已取消 (ref可在异步期间被cancelDownload修改)
-            if (hasBeenCancelled()) {
-              return;
-            }
-            dispatch({ type: "SET_FOLDER_PROGRESS", progress: Math.round(metadata.percent) });
+          onFileError: (file, error) => {
+            logger.error(`文件 ${file.path} 下载失败:`, error);
           },
-        );
+        });
 
-        // 最后一次检查是否已取消 (ref可在异步期间被cancelDownload修改)
-        if (hasBeenCancelled()) {
-          logger.info("文件夹下载已取消");
-          return;
+        if (!hasBeenCancelled()) {
+          dispatch({ type: "SET_PROCESSING_FILES", count: processedCount });
+          dispatch({ type: "SET_FOLDER_PROGRESS", progress: 100 });
         }
 
-        saveAs(zipBlob, `${folderName}.zip`);
         logger.info(`文件夹下载完成: ${path}`);
       } catch (e: unknown) {
         const error = e as Error;
-        if (error.name === "AbortError" || hasBeenCancelled()) {
+        if (outputSink !== null && !zipPipelineStarted) {
+          await outputSink.abort(error);
+        }
+
+        if (isAbortError(error) || hasBeenCancelled()) {
           logger.info("文件夹下载已取消");
         } else {
           logger.error("下载文件夹失败:", error);

src/services/github/RequestBatcher.test.ts

@@ -1,4 +1,5 @@
 import { beforeEach, describe, expect, it, vi } from "vite-plus/test";
+import { createAbortError } from "@/utils/network/abort";
 
 vi.mock("@/utils", () => ({
   logger: {
@@ -97,4 +98,20 @@ describe("RequestBatcher", () => {
     expect(firstResult).toEqual({ value: 1 });
     expect(secondResult).toEqual({ value: 1 });
   });
+
+  it("does not retry aborted requests", async () => {
+    const batcher = new RequestBatcher();
+    const executeRequest = vi.fn(async () => {
+      throw createAbortError("Request aborted");
+    });
+
+    await expect(
+      batcher.enqueue("https://example.com/repos", executeRequest, {
+        method: "GET",
+        headers: { Accept: "application/json" },
+      }),
+    ).rejects.toMatchObject({ name: "AbortError" });
+
+    expect(executeRequest).toHaveBeenCalledTimes(1);
+  });
 });

src/services/github/RequestBatcher.ts

@@ -1,4 +1,5 @@
 import { logger } from "@/utils";
+import { isAbortError } from "@/utils/network/abort";
 import { createTimeWheel } from "@/utils/data-structures/TimeWheel";
 import type { TimeWheel } from "@/utils/data-structures/TimeWheel";
 
@@ -205,7 +206,15 @@ export class RequestBatcher {
 
       // 如果是队列中的第一个请求，执行它
       if (isFirstRequest) {
-        void this.executeWithRetry(requestKey, key, executeRequest, fingerprint, fingerprintCache);
+        void this.executeWithRetry(
+          requestKey,
+          key,
+          executeRequest,
+          fingerprint,
+          fingerprintCache,
+        ).catch(() => {
+          // 请求结果会通过队列中的 promise 向调用方传递，这里只避免未处理 rejection。
+        });
       }
     });
   }
@@ -257,6 +266,7 @@ export class RequestBatcher {
     const retryOptions: RetryOptions = {
       maxRetries: this.maxRetries,
       backoff: (attempt) => Math.min(1000 * Math.pow(2, attempt), 5000), // 指数退避，最大5秒
+      shouldRetry: (error) => !isAbortError(error),
       onRetry: (attempt, error) => {
         const errorMessage = error instanceof Error ? error.message : String(error);
         logger.warn(

src/services/github/core/content/service.test.ts

@@ -0,0 +1,149 @@
+import { beforeEach, describe, expect, it, vi } from "vite-plus/test";
+
+import { createAbortError } from "@/utils/network/abort";
+
+const { axiosGetMock } = vi.hoisted(() => ({
+  axiosGetMock: vi.fn(),
+}));
+
+vi.mock("axios", () => ({
+  default: {
+    get: axiosGetMock,
+  },
+}));
+
+vi.mock("@/utils", () => ({
+  logger: {
+    debug: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+  },
+}));
+
+vi.mock("../../config", () => ({
+  getForceServerProxy: vi.fn(() => false),
+  shouldUseServerAPI: vi.fn(() => false),
+}));
+
+vi.mock("../Auth", () => ({
+  getAuthHeaders: vi.fn(() => ({ Authorization: "Bearer test-token" })),
+}));
+
+vi.mock("../Config", () => ({
+  USE_TOKEN_MODE: false,
+  getApiUrl: vi.fn((path: string, branch: string) => `https://api.example.com/${branch}/${path}`),
+  getCurrentBranch: vi.fn(() => "main"),
+}));
+
+vi.mock("../../schemas", () => ({
+  safeValidateGitHubContentsResponse: vi.fn((data: unknown) => ({ success: true, data })),
+  filterAndNormalizeGitHubContents: vi.fn((data: unknown) => data),
+  transformGitHubContentsResponse: vi.fn((data: unknown) => data),
+  validateGitHubContentsArray: vi.fn(() => ({ isValid: true, invalidItems: [] })),
+}));
+
+vi.mock("./cacheState", () => ({
+  ensureCacheInitialized: vi.fn(async () => {}),
+  getCachedDirectoryContents: vi.fn(async () => null),
+  getCachedFileContent: vi.fn(async () => null),
+  isCacheAvailable: vi.fn(() => false),
+  storeDirectoryContents: vi.fn(async () => {}),
+  storeFileContent: vi.fn(async () => {}),
+}));
+
+vi.mock("./cacheKeys", () => ({
+  buildContentsCacheKey: vi.fn((path: string, branch: string) => `${branch}:${path}`),
+}));
+
+vi.mock("./hydrationStore", () => ({
+  consumeHydratedDirectory: vi.fn(async () => null),
+  consumeHydratedFile: vi.fn(async () => null),
+  hydrateInitialContent: vi.fn(),
+  INITIAL_CONTENT_EXCLUDE_FILES: [],
+}));
+
+if (typeof window === "undefined") {
+  vi.stubGlobal("window", globalThis);
+}
+
+import { shouldUseServerAPI } from "../../config";
+const { clearBatcherCache, getContents } = await import("./service");
+
+describe("content service abort handling", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.unstubAllGlobals();
+    vi.stubGlobal("window", globalThis);
+    clearBatcherCache();
+    vi.mocked(shouldUseServerAPI).mockReturnValue(false);
+  });
+
+  it("propagates abort to direct fetch requests", async () => {
+    let fetchSignal: AbortSignal | undefined;
+    let resolveFetchStarted: (() => void) | null = null;
+    const fetchStarted = new Promise<void>((resolve) => {
+      resolveFetchStarted = resolve;
+    });
+    const fetchMock = vi.fn((_: RequestInfo | URL, init?: RequestInit) => {
+      fetchSignal = init?.signal;
+      resolveFetchStarted?.();
+
+      return new Promise<Response>((_, reject) => {
+        fetchSignal?.addEventListener("abort", () => reject(createAbortError("Request aborted")), {
+          once: true,
+        });
+      });
+    });
+    const controller = new AbortController();
+    vi.stubGlobal("fetch", fetchMock);
+
+    const promise = getContents("docs", controller.signal);
+    await fetchStarted;
+    controller.abort();
+
+    await expect(promise).rejects.toMatchObject({ name: "AbortError" });
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+    expect(fetchSignal?.aborted).toBe(true);
+  });
+
+  it("propagates abort to server proxy axios requests", async () => {
+    vi.mocked(shouldUseServerAPI).mockReturnValue(true);
+    const controller = new AbortController();
+    let resolveAxiosStarted: (() => void) | null = null;
+    const axiosStarted = new Promise<void>((resolve) => {
+      resolveAxiosStarted = resolve;
+    });
+
+    axiosGetMock.mockImplementationOnce(async (_url, config) => {
+      const signal = config?.signal as AbortSignal | undefined;
+      resolveAxiosStarted?.();
+
+      return new Promise((_, reject) => {
+        signal?.addEventListener(
+          "abort",
+          () =>
+            reject(
+              Object.assign(new Error("canceled"), {
+                name: "CanceledError",
+                code: "ERR_CANCELED",
+              }),
+            ),
+          { once: true },
+        );
+      });
+    });
+
+    const promise = getContents("docs", controller.signal);
+    await axiosStarted;
+    controller.abort();
+
+    await expect(promise).rejects.toMatchObject({ name: "AbortError" });
+    expect(axiosGetMock).toHaveBeenCalledWith(
+      "/api/github?action=getContents&path=docs&branch=main",
+      {
+        signal: controller.signal,
+      },
+    );
+  });
+});