Skip to content

Commit 074be01

Browse files
cyfung1031CodFrm
cyfung1031
and
CodFrm
authored
🐛(sync) (Codex) Sync digest consistency / eventual consistency (#1393)
* fix(sync): Sync digest consistency / eventual consistency * fix(sync): preserve cloud-native digest in updateFileDigest 云端 list 已能返回原生 digest 时不应被本地 md5 覆盖。webdav/onedrive/s3 返回 etag、dropbox 返回 content_hash，与 md5OfText 格式不一致，强制覆盖 会让下次同步比对必失败，导致未变动脚本被反复识别为已变动。改为只在云端 列表暂时漏掉刚上传的文件时用本地 md5 兜底。 --------- Co-authored-by: 王一之 <yz@ggnb.top>
1 parent 27006fe commit 074be01

2 files changed

Lines changed: 141 additions & 24 deletions

File tree

‎src/app/service/service_worker/synchronize.test.ts‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { initTestEnv } from "@Tests/utils";
44
import type FileSystem from "@Packages/filesystem/filesystem";
55
import type { CloudSyncConfig } from "@App/pkg/config/config";
66
import { stackAsyncTask } from "@App/pkg/utils/async_queue";
7+
import { md5OfText } from "@App/pkg/utils/crypto";
78

89
initTestEnv();
910

@@ -397,6 +398,101 @@ console.log("ok");`
397398
expect(order.indexOf("push:end")).toBeLessThan(order.indexOf("digest:list"));
398399
});
399400

401+
it("keeps pushed script digest when cloud list is stale after upload", async () => {
402+
const scriptCode = "// code";
403+
const script = {
404+
uuid: "push-uuid",
405+
name: "push",
406+
origin: "origin",
407+
downloadUrl: "download-url",
408+
checkUpdateUrl: "check-update-url",
409+
updatetime: 1,
410+
createtime: 1,
411+
status: 1,
412+
sort: 0,
413+
metadata: {},
414+
};
415+
const fs = createFs({
416+
list: vi.fn().mockResolvedValueOnce([]).mockResolvedValueOnce([]),
417+
});
418+
const service = new SynchronizeService(
419+
{} as any,
420+
{} as any,
421+
{} as any,
422+
{} as any,
423+
{} as any,
424+
{} as any,
425+
{} as any,
426+
{
427+
scriptCodeDAO: {
428+
get: vi.fn().mockResolvedValue({ code: scriptCode }),
429+
},
430+
all: vi.fn().mockResolvedValue([script]),
431+
} as any
432+
);
433+
434+
await service.syncOnce({ ...syncConfig, syncStatus: false }, fs);
435+
436+
const metaJson = JSON.stringify({
437+
uuid: script.uuid,
438+
origin: script.origin,
439+
downloadUrl: script.downloadUrl,
440+
checkUpdateUrl: script.checkUpdateUrl,
441+
});
442+
await expect((service as any).storage.get("file_digest")).resolves.toEqual({
443+
"push-uuid.user.js": md5OfText(scriptCode),
444+
"push-uuid.meta.json": md5OfText(metaJson),
445+
});
446+
});
447+
448+
it("preserves cloud-native digest and does not overwrite with pushed md5", async () => {
449+
// 各后端 digest 格式不一致（webdav/onedrive 是 etag、dropbox 是 content_hash 等），
450+
// 上传后再次 list 已经能拿到原生 digest 时，必须保留它，不能被本地 md5 覆盖，
451+
// 否则下次同步比对会因格式不一致而把未变动的脚本判定为已变动并触发不必要的拉取/推送
452+
const scriptCode = "// code";
453+
const script = {
454+
uuid: "push-uuid",
455+
name: "push",
456+
origin: "origin",
457+
downloadUrl: "download-url",
458+
checkUpdateUrl: "check-update-url",
459+
updatetime: 1,
460+
createtime: 1,
461+
status: 1,
462+
sort: 0,
463+
metadata: {},
464+
};
465+
const cloudListAfterPush = [
466+
{ name: "push-uuid.user.js", digest: "etag-user-js", updatetime: 1 },
467+
{ name: "push-uuid.meta.json", digest: "etag-meta-json", updatetime: 1 },
468+
];
469+
const fs = createFs({
470+
list: vi.fn().mockResolvedValueOnce([]).mockResolvedValueOnce(cloudListAfterPush),
471+
});
472+
const service = new SynchronizeService(
473+
{} as any,
474+
{} as any,
475+
{} as any,
476+
{} as any,
477+
{} as any,
478+
{} as any,
479+
{} as any,
480+
{
481+
scriptCodeDAO: {
482+
get: vi.fn().mockResolvedValue({ code: scriptCode }),
483+
},
484+
all: vi.fn().mockResolvedValue([script]),
485+
} as any
486+
);
487+
488+
await service.syncOnce({ ...syncConfig, syncStatus: false }, fs);
489+
490+
await expect((service as any).storage.get("file_digest")).resolves.toEqual({
491+
"push-uuid.user.js": "etag-user-js",
492+
"push-uuid.meta.json": "etag-meta-json",
493+
});
494+
});
495+
400496
it("scriptInstall enters cloud_sync queue and updates digest after push", async () => {
401497
let releaseSync!: () => void;
402498
const syncGate = new Promise<void>((resolve) => {
@@ -439,6 +535,7 @@ console.log("ok");`
439535
vi.spyOn(service as any, "buildFileSystem").mockResolvedValue(installFs);
440536
vi.spyOn(service, "pushScript").mockImplementation(async () => {
441537
order.push("install:push");
538+
return {};
442539
});
443540
const realUpdateDigest = service.updateFileDigest.bind(service);
444541
vi.spyOn(service, "updateFileDigest").mockImplementation(async (fs) => {

‎src/app/service/service_worker/synchronize.ts‎

Lines changed: 44 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import { dayFormat } from "@App/pkg/utils/day_format";
3434
import i18n, { i18nName } from "@App/locales/locales";
3535
import { InfoNotification } from "./utils";
3636
import { stackAsyncTask } from "@App/pkg/utils/async_queue";
37+
import { md5OfText } from "@App/pkg/utils/crypto";
3738

3839
// type SynchronizeTarget = "local";
3940

@@ -67,6 +68,10 @@ type ScriptcatSyncStatus = {
6768

6869
type PushScriptParam = TInstallScriptParams;
6970

71+
type FileDigestMap = {
72+
[key: string]: string;
73+
};
74+
7075
const SYNC_SERVICE_TASK_KEY = "cloud_sync_queue";
7176

7277
export class SynchronizeService {
@@ -348,10 +353,7 @@ export class SynchronizeService {
348353
// 根据文件名生成一个map
349354
const uuidMap = new Map<string, Partial<SyncFiles>>();
350355
// 储存文件摘要,用于检测文件是否有变化
351-
const fileDigestMap =
352-
((await this.storage.get("file_digest")) as {
353-
[key: string]: string;
354-
}) || {};
356+
const fileDigestMap = ((await this.storage.get("file_digest")) as FileDigestMap) || {};
355357

356358
for (const file of list) {
357359
if (file.name.endsWith(".user.js")) {
@@ -397,7 +399,7 @@ export class SynchronizeService {
397399
}
398400

399401
// 对比脚本列表和文件列表,进行同步
400-
const result: Promise<void>[] = [];
402+
const result: Promise<FileDigestMap | void>[] = [];
401403
const updateScript: Map<string, boolean> = new Map();
402404
// 记录被跳过的孤儿云端脚本（仅 .user.js 无 .meta.json）
403405
// 避免本机回写 scriptcat-sync.json 时丢失对应 uuid 的云端 status
@@ -426,7 +428,7 @@ export class SynchronizeService {
426428
} else {
427429
// 否则认为是一个无效的.meta文件，进行删除，并进行同步
428430
await fs.delete(file.meta!.name);
429-
await this.pushScript(fs, script);
431+
return await this.pushScript(fs, script);
430432
}
431433
})()
432434
);
@@ -469,7 +471,13 @@ export class SynchronizeService {
469471
result.push(this.pushScript(fs, script));
470472
});
471473
// 忽略错误
472-
await Promise.allSettled(result);
474+
const syncResults = await Promise.allSettled(result);
475+
const pushedFileDigestMap: FileDigestMap = {};
476+
syncResults.forEach((ret) => {
477+
if (ret.status === "fulfilled" && ret.value) {
478+
Object.assign(pushedFileDigestMap, ret.value);
479+
}
480+
});
473481
// 同步状态
474482
if (syncConfig.syncStatus) {
475483
const scriptlist = await this.scriptDAO.all();
@@ -533,17 +541,25 @@ export class SynchronizeService {
533541
}
534542
// 重新获取文件列表,保存文件摘要
535543
this.logger.info("update file digest");
536-
await this.updateFileDigest(fs);
544+
await this.updateFileDigest(fs, pushedFileDigestMap);
537545
this.logger.info("sync complete");
538546
return;
539547
}
540548

541-
async updateFileDigest(fs: FileSystem) {
549+
async updateFileDigest(fs: FileSystem, knownFileDigestMap: FileDigestMap = {}) {
542550
const newList = await fs.list();
543-
const newFileDigestMap: { [key: string]: string } = {};
551+
const newFileDigestMap: FileDigestMap = {};
544552
for (const file of newList) {
545553
newFileDigestMap[file.name] = file.digest;
546554
}
555+
// 各后端 digest 格式不一（WebDAV/OneDrive/S3 是 etag、Dropbox 是 content_hash、Zip 为空，
556+
// 仅 GoogleDrive/Baidu 是 md5），只在云端列表暂时漏掉刚上传的文件时用本地 md5 兜底，
557+
// 不能覆盖 fs.list 已返回的原生 digest，否则下次同步比对会因格式不一致而误判
558+
for (const name in knownFileDigestMap) {
559+
if (!(name in newFileDigestMap)) {
560+
newFileDigestMap[name] = knownFileDigestMap[name];
561+
}
562+
}
547563
await this.storage.set("file_digest", newFileDigestMap);
548564
return;
549565
}
@@ -581,8 +597,9 @@ export class SynchronizeService {
581597
}
582598

583599
// 上传脚本
584-
async pushScript(fs: FileSystem, script: PushScriptParam) {
600+
async pushScript(fs: FileSystem, script: PushScriptParam): Promise<FileDigestMap> {
585601
const filename = `${script.uuid}.user.js`;
602+
const metaFilename = `${script.uuid}.meta.json`;
586603
const logger = this.logger.with({
587604
uuid: script.uuid,
588605
name: script.name,
@@ -592,22 +609,25 @@ export class SynchronizeService {
592609
const w = await fs.create(filename);
593610
// 获取脚本代码
594611
const code = await this.scriptCodeDAO.get(script.uuid);
595-
await w.write(code!.code);
596-
const meta = await fs.create(`${script.uuid}.meta.json`);
597-
await meta.write(
598-
JSON.stringify(<SyncMeta>{
599-
uuid: script.uuid,
600-
origin: script.origin,
601-
downloadUrl: script.downloadUrl,
602-
checkUpdateUrl: script.checkUpdateUrl,
603-
})
604-
);
612+
const scriptCode = code!.code;
613+
await w.write(scriptCode);
614+
const meta = await fs.create(metaFilename);
615+
const metaJson = JSON.stringify(<SyncMeta>{
616+
uuid: script.uuid,
617+
origin: script.origin,
618+
downloadUrl: script.downloadUrl,
619+
checkUpdateUrl: script.checkUpdateUrl,
620+
});
621+
await meta.write(metaJson);
605622
logger.info("push script success");
623+
return {
624+
[filename]: md5OfText(scriptCode),
625+
[metaFilename]: md5OfText(metaJson),
626+
};
606627
} catch (e) {
607628
logger.error("push script error", Logger.E(e));
608629
throw e;
609630
}
610-
return;
611631
}
612632

613633
async pullScript(fs: FileSystem, file: SyncFiles, status: ScriptcatSyncStatus | undefined, existingScript?: Script) {
@@ -703,8 +723,8 @@ export class SynchronizeService {
703723
if (config.enable) {
704724
stackAsyncTask(SYNC_SERVICE_TASK_KEY, async () => {
705725
const fs = await this.buildFileSystem(config);
706-
await this.pushScript(fs, params.script);
707-
await this.updateFileDigest(fs);
726+
const pushedFileDigestMap = await this.pushScript(fs, params.script);
727+
await this.updateFileDigest(fs, pushedFileDigestMap);
708728
}).catch((e) => {
709729
this.logger.error("push script on install error", Logger.E(e));
710730
});

0 commit comments

Comments
 (0)