🐛 解决并发 xhr 的 session rule 数量问题 (#1353)

* 解决并发 xhr 的 session rule 数量问题

* typo

* code fix

* fix mock

* update unit test

* 改善 session rule 移除处理 及 scXhrRequests 处理

* fix mock

* 处理 code review 反馈: 修复并发锁/错误回滚并改善测试

- dnr_id_controller: nextSessionRuleId 改为 while 循环反复判定上限，
  removeSessionRuleIdEntry 释放一次只放行一个 waiter，避免撑爆 LIMIT
- dnr_id_controller: 初始化时用历史最大 rule id 更新 SESSION_RULE_ID_BEGIN
- gm_api: headersSettled 在 lastError 时不释放 ruleID 避免本地/浏览器状态不一致
- gm_api: updateSessionRules addRules 失败时回滚 headerModifierMap 与 ruleId
- gm_api: 修正注释 headerModifer -> headerModifier
- chrome-extension-mock: onResponseStarted.addListener 暴露 callback 供测试触发
- dnr_id_controller.test: describe/it 标题改为中文，去除 Math.random
- dnr_id_controller.test: 并发用例改为 "1 次释放 = 1 个 waiter 放行" 语义
- vitest.setup: 移除注释掉的 spyOn 调试代码

---------

Co-authored-by: 王一之 <yz@ggnb.top>

Author: cyfung1031

packages/chrome-extension-mock/declarativ_net_request.ts

@@ -1,4 +1,8 @@
 export default class DeclarativeNetRequest {
+  MAX_NUMBER_OF_SESSION_RULES = 5000;
+
+  private _sessionRules: chrome.declarativeNetRequest.Rule[] = [];
+
   HeaderOperation = {
     APPEND: "append",
     SET: "set",
@@ -30,9 +34,44 @@ export default class DeclarativeNetRequest {
     OTHER: "other",
   };
 
-  updateSessionRules() {
+  updateSessionRules(arg1: any, arg2: any): Promise<void> {
+    let options: {
+      addRules?: chrome.declarativeNetRequest.Rule[];
+      removeRuleIds?: number[];
+    } = {};
+    let callback: undefined | ((...args: any) => any) = undefined;
+
+    if (typeof arg1 === "function") {
+      callback = arg1;
+    } else if (typeof arg2 === "function") {
+      callback = arg2;
+    }
+    if (typeof arg1 === "object" && arg1) options = arg1;
+
     return new Promise<void>((resolve) => {
+      const { addRules = [], removeRuleIds = [] } = options;
+
+      // Remove rules by ID
+      if (removeRuleIds.length > 0) {
+        this._sessionRules = this._sessionRules.filter((rule) => !removeRuleIds.includes(rule.id));
+      }
+
+      // Add or update rules (upsert by ID)
+      for (const newRule of addRules) {
+        const existingIndex = this._sessionRules.findIndex((rule) => rule.id === newRule.id);
+        if (existingIndex !== -1) {
+          this._sessionRules[existingIndex] = newRule; // update
+        } else {
+          this._sessionRules.push(newRule); // add
+        }
+      }
+
       resolve();
+      callback?.();
     });
   }
+
+  getSessionRules(): Promise<chrome.declarativeNetRequest.Rule[]> {
+    return Promise.resolve([...this._sessionRules]);
+  }
 }

packages/chrome-extension-mock/web_reqeuest.ts

@@ -2,6 +2,7 @@ import EventEmitter from "eventemitter3";
 
 export default class WebRequest {
   sendHeader?: (details: chrome.webRequest.OnSendHeadersDetails) => chrome.webRequest.BlockingResponse | void;
+  responseStarted?: (details: chrome.webRequest.OnResponseStartedDetails) => void;
 
   onBeforeSendHeaders = {
     addListener: (callback: any) => {
@@ -15,6 +16,12 @@ export default class WebRequest {
     },
   };
 
+  onResponseStarted = {
+    addListener: (callback: any) => {
+      this.responseStarted = callback;
+    },
+  };
+
   onCompleted = {
     addListener: () => {
       // TODO

src/app/service/service_worker/gm_api/dnr_id_controller.test.ts

@@ -0,0 +1,217 @@
+import { sleep } from "@App/pkg/utils/utils";
+import { describe, expect, it } from "vitest";
+import {
+  getSessionRuleIds,
+  LIMIT_SESSION_RULES,
+  nextSessionRuleId,
+  removeSessionRuleIdEntry,
+} from "./dnr_id_controller";
+
+describe("getSessionRuleIds", () => {
+  it("从现有 chrome session rules 初始化", async () => {
+    const ids = await getSessionRuleIds();
+    expect(ids.size).lessThan(100);
+    await nextSessionRuleId();
+    expect(ids.size).greaterThanOrEqual(1);
+    await nextSessionRuleId();
+    expect(ids.size).greaterThanOrEqual(2);
+  });
+});
+
+describe("nextSessionRuleId", () => {
+  it("每次调用返回唯一递增的 id", async () => {
+    const id1 = await nextSessionRuleId();
+    const id2 = await nextSessionRuleId();
+    const id3 = await nextSessionRuleId();
+
+    expect(id2).toBeGreaterThan(id1);
+    expect(id3).toBeGreaterThan(id2);
+  });
+
+  it("跳过已存在于 session rules 中的 id", async () => {
+    const ids = await getSessionRuleIds();
+    const next = await nextSessionRuleId();
+
+    ids.add(next + 1);
+    const skipped = await nextSessionRuleId();
+
+    expect(skipped).toBeGreaterThan(next + 1);
+  });
+});
+
+describe("removeSessionRuleIdEntry", () => {
+  it("从追踪集合中移除指定 id", async () => {
+    const ids = await getSessionRuleIds();
+    const id = await nextSessionRuleId();
+
+    ids.add(id);
+    removeSessionRuleIdEntry(id);
+
+    expect(ids.has(id)).toBe(false);
+  });
+
+  it("sessionRuleIds 未初始化时为 no-op", () => {
+    expect(() => removeSessionRuleIdEntry(10404)).not.toThrow();
+  });
+
+  it("ruleId <= 10000 时抛错", () => {
+    expect(() => removeSessionRuleIdEntry(10000)).toThrow();
+    expect(() => removeSessionRuleIdEntry(1)).toThrow();
+  });
+
+  it("回退 SESSION_RULE_ID_BEGIN 使被移除的 id 下次被复用", async () => {
+    const ids = await getSessionRuleIds();
+
+    const id1 = await nextSessionRuleId();
+    const id2 = await nextSessionRuleId();
+    const id3 = await nextSessionRuleId();
+
+    ids.add(id1);
+    ids.add(id2);
+    ids.add(id3);
+
+    // 移除最新的 id，counter 回退并复用该 id
+    removeSessionRuleIdEntry(id3);
+    const reused = await nextSessionRuleId();
+    expect(reused).toBe(id3);
+  });
+
+  it("移除的 id 在 counter 之前时仍会回退", async () => {
+    const ids = await getSessionRuleIds();
+
+    const id1 = await nextSessionRuleId();
+    const id2 = await nextSessionRuleId();
+    const id3 = await nextSessionRuleId();
+
+    ids.add(id1);
+    ids.add(id2);
+    ids.add(id3);
+
+    // 移除较早的 id，counter 回退到 id1 - 1，下次分配会得到 id1
+    removeSessionRuleIdEntry(id1);
+    const reused = await nextSessionRuleId();
+    expect(reused).toBe(id1);
+  });
+
+  it("移除 counter 之后的 id 不回退 counter", async () => {
+    const id1 = await nextSessionRuleId();
+    const id2 = await nextSessionRuleId();
+    const id3 = await nextSessionRuleId();
+    const _id4 = await nextSessionRuleId();
+    const id5 = await nextSessionRuleId();
+    const id6 = await nextSessionRuleId();
+
+    removeSessionRuleIdEntry(id1);
+    removeSessionRuleIdEntry(id5);
+    removeSessionRuleIdEntry(id3);
+    // removeSessionRuleIdEntry(id4);
+    removeSessionRuleIdEntry(id2);
+    const nextBefore = await nextSessionRuleId(); // e.g. 10001
+    removeSessionRuleIdEntry(id6);
+
+    expect(await nextSessionRuleId()).toBe(nextBefore + 1); // counter 未变，正常递增
+    expect(await nextSessionRuleId()).toBe(nextBefore + 2);
+    // expect(await nextSessionRuleId()).toBe(nextBefore + 3);
+    expect(await nextSessionRuleId()).toBe(nextBefore + 4);
+    expect(await nextSessionRuleId()).toBe(nextBefore + 5);
+  });
+});
+
+describe("nextSessionRuleId limit control", () => {
+  it("达到上限时锁定，移除条目后解锁", async () => {
+    const ids = await getSessionRuleIds();
+    expect(ids.size).toBeLessThan(100);
+
+    const added = [];
+    for (let w = ids.size; w < LIMIT_SESSION_RULES; w++) {
+      const j = await nextSessionRuleId();
+      added.push(j);
+      ids.add(j);
+    }
+    expect(ids.size).toBeGreaterThan(1000);
+
+    const lockedPromise = nextSessionRuleId();
+
+    const raceResult1 = await Promise.race([lockedPromise.then(() => "resolved"), sleep(5).then(() => "pending")]);
+    expect(raceResult1).toBe("pending");
+
+    // 使用固定索引而非随机，保证测试可重复
+    const p1 = added[0];
+    const p2 = added[6];
+    removeSessionRuleIdEntry(p1);
+    removeSessionRuleIdEntry(p2);
+
+    const raceResult2 = await Promise.race([lockedPromise.then(() => "resolved"), sleep(5).then(() => "pending")]);
+    expect(raceResult2).toBe("resolved");
+
+    const id1 = await lockedPromise;
+    const id2 = await nextSessionRuleId();
+    expect(id1).toBe(p1);
+    expect(id2).toBe(p2);
+
+    for (const k of added) {
+      removeSessionRuleIdEntry(k);
+    }
+
+    const res = await getSessionRuleIds();
+    expect(res).toBe(ids);
+    expect(res.size).toBeLessThan(100);
+  });
+
+  it("单次移除仅放行 1 个 waiter，其余继续等待", async () => {
+    const ids = await getSessionRuleIds();
+    expect(ids.size).toBeLessThan(100);
+
+    const added = [];
+    for (let w = ids.size; w < LIMIT_SESSION_RULES; w++) {
+      const j = await nextSessionRuleId();
+      added.push(j);
+      ids.add(j);
+    }
+    expect(ids.size).toBeGreaterThan(1000);
+
+    // 在已达上限时发起多个并发调用
+    const p1 = nextSessionRuleId();
+    const p2 = nextSessionRuleId();
+    const p3 = nextSessionRuleId();
+
+    const raceResult = await Promise.race([
+      Promise.all([p1, p2, p3]).then(() => "resolved"),
+      sleep(5).then(() => "pending"),
+    ]);
+    expect(raceResult).toBe("pending");
+
+    // 单次释放 1 个 slot: 只应放行 1 个 waiter，剩余仍挂起
+    removeSessionRuleIdEntry(added[0]);
+    const firstResolved = await Promise.race([p1.then(() => "resolved"), sleep(50).then(() => "pending")]);
+    expect(firstResolved).toBe("resolved");
+
+    const remainingStillPending = await Promise.race([
+      Promise.all([p2, p3]).then(() => "resolved"),
+      sleep(5).then(() => "pending"),
+    ]);
+    expect(remainingStillPending).toBe("pending");
+
+    // 继续释放 2 个 slot，剩余 waiter 才能继续完成
+    removeSessionRuleIdEntry(added[1]);
+    removeSessionRuleIdEntry(added[2]);
+    const results = await Promise.race([Promise.all([p2, p3]).then((vals) => vals), sleep(50).then(() => null)]);
+    expect(results).not.toBeNull();
+
+    // 任何时候 size 都不应超过 LIMIT_SESSION_RULES
+    expect(ids.size).toBeLessThanOrEqual(LIMIT_SESSION_RULES);
+
+    for (const k of added) {
+      if (ids.has(k)) removeSessionRuleIdEntry(k);
+    }
+    // p1/p2/p3 分配的 id 也要清理
+    const allocated = [await p1, ...(results as number[])];
+    for (const k of allocated) {
+      if (ids.has(k)) removeSessionRuleIdEntry(k);
+    }
+
+    const res = await getSessionRuleIds();
+    expect(res).toBe(ids);
+    expect(res.size).toBeLessThan(100);
+  });
+});

src/app/service/service_worker/gm_api/dnr_id_controller.ts

@@ -0,0 +1,66 @@
+import type { Deferred } from "@App/pkg/utils/utils";
+import { deferred } from "@App/pkg/utils/utils";
+
+let sessionRuleIdsPromise: Promise<Set<number>> | null = null;
+let sessionRuleIds: Set<number> | null = null;
+
+let SESSION_RULE_ID_BEGIN = 10000;
+export const LIMIT_SESSION_RULES =
+  process.env.VI_TESTING === "true" ? 1234 : chrome.declarativeNetRequest.MAX_NUMBER_OF_SESSION_RULES - 300;
+let lockSessionRuleCreation: Deferred<void> | null = null;
+
+export const getSessionRuleIds = async (): Promise<Set<number>> => {
+  if (!sessionRuleIdsPromise) {
+    sessionRuleIdsPromise = chrome.declarativeNetRequest
+      .getSessionRules()
+      .then((rules) => {
+        const existingRuleIds = rules.map((rule) => rule.id).filter(Boolean);
+        // 根据历史 session rule 的最大 id 更新 SESSION_RULE_ID_BEGIN
+        // 避免 SW 重启后从 10001 起做大量 do/while 递增扫描
+        if (existingRuleIds.length > 0) {
+          SESSION_RULE_ID_BEGIN = Math.max(SESSION_RULE_ID_BEGIN, ...existingRuleIds);
+        }
+        sessionRuleIds = new Set(existingRuleIds);
+        return sessionRuleIds;
+      })
+      .catch((e) => {
+        console.warn(e);
+        sessionRuleIds = new Set<number>([]);
+        return sessionRuleIds;
+      });
+  }
+  const ruleIds = sessionRuleIds || (await sessionRuleIdsPromise);
+  return ruleIds;
+};
+
+export const removeSessionRuleIdEntry = (ruleId: number) => {
+  if (ruleId <= 10000) {
+    throw new Error("removeSessionRuleIdEntry cannot remove ids not created by nextSessionRuleId");
+  }
+  if (sessionRuleIds) {
+    if (sessionRuleIds.delete(ruleId) === true) {
+      if (ruleId <= SESSION_RULE_ID_BEGIN + 1) {
+        SESSION_RULE_ID_BEGIN = ruleId - 1;
+      }
+      // 唤醒所有等待者: 每个等待者会在 while 循环内重新判定 size 并决定是继续分配还是再次等待
+      // 这样 "1 次释放 => 1 个等待者通过"，避免 N 个等待者同时放行再次撑爆上限
+      lockSessionRuleCreation?.resolve();
+      lockSessionRuleCreation = null;
+    }
+  }
+};
+
+export const nextSessionRuleId = async () => {
+  const ruleIds = await getSessionRuleIds();
+  // 用 while 循环反复判定上限: 等待者被唤醒后如果 slot 已被其他等待者抢占，会再次进入等待
+  while (ruleIds.size + 1 > LIMIT_SESSION_RULES) {
+    if (!lockSessionRuleCreation) lockSessionRuleCreation = deferred<void>();
+    await lockSessionRuleCreation.promise;
+  }
+  let id;
+  do {
+    id = ++SESSION_RULE_ID_BEGIN;
+  } while (ruleIds.has(id));
+  ruleIds.add(id);
+  return id;
+};