♻️ 新储存引擎数据迁移

Author: CodFrm

src/app/migrate.ts

@@ -1,8 +1,12 @@
+import { getStorageName } from "@App/pkg/utils/utils";
 import { db } from "./repo/dao";
-import { Script } from "./repo/scripts";
+import { Script, ScriptAndCode, ScriptCodeDAO, ScriptDAO } from "./repo/scripts";
+import { Subscribe, SubscribeDAO } from "./repo/subscribe";
+import { Value, ValueDAO } from "./repo/value";
+import { Permission, PermissionDAO } from "./repo/permission";
 
 // 0.10.0重构,重命名字段,统一使用小峰驼
-function renameField(): void {
+function renameField() {
   db.version(16)
     .stores({
       scripts:
@@ -32,10 +36,199 @@ function renameField(): void {
     // export是0.10.x时的兼容性处理
     export: "++id,&scriptId",
   });
+  const v = 36;
   // 将脚本数据迁移到chrome.storage
-  // db.version(18)
-  //   .stores({})
-  //   .upgrade((tx) => {});
+  db.version(v).upgrade(() => {
+    // 默认使用的事务，这里加个延时，用db.open()打开数据库后，再执行
+    setTimeout(async () => {
+      try {
+        // 迁移脚本
+        const scripts = await db.table("scripts").toArray();
+        const scriptDAO = new ScriptDAO();
+        const scriptCodeDAO = new ScriptCodeDAO();
+        console.log("开始迁移脚本数据", scripts.length);
+        await Promise.all(
+          scripts.map(async (script: ScriptAndCode) => {
+            const {
+              uuid,
+              name,
+              namespace,
+              author,
+              originDomain,
+              subscribeUrl,
+              type,
+              sort,
+              status,
+              runStatus,
+              metadata,
+              createtime,
+              checktime,
+              code,
+              checkUpdateUrl,
+              downloadUrl,
+              selfMetadata,
+              config,
+              error,
+              updatetime,
+              lastruntime,
+              nextruntime,
+            } = script;
+            const s = await scriptDAO.save({
+              uuid,
+              name,
+              namespace,
+              author,
+              originDomain,
+              origin,
+              checkUpdateUrl,
+              downloadUrl,
+              metadata,
+              selfMetadata,
+              subscribeUrl,
+              config,
+              type,
+              status,
+              sort,
+              runStatus,
+              error,
+              createtime,
+              updatetime,
+              checktime,
+              lastruntime,
+              nextruntime,
+            });
+            return scriptCodeDAO
+              .save({
+                uuid: s.uuid,
+                code,
+              })
+              .catch((e) => {
+                console.log("脚本代码迁移失败", e);
+                return Promise.reject(e);
+              });
+          })
+        );
+        // 迁移订阅
+        const subscribe = await db.table("subscribe").toArray();
+        const subscribeDAO = new SubscribeDAO();
+        if (subscribe.length) {
+          await Promise.all(
+            subscribe.map((s: Subscribe) => {
+              console.log("1234", s);
+              const { url, name, code, author, scripts, metadata, status, createtime, updatetime, checktime } = s;
+              return subscribeDAO.save({
+                url,
+                name,
+                code,
+                author,
+                scripts,
+                metadata,
+                status,
+                createtime,
+                updatetime,
+                checktime,
+              });
+            })
+          );
+        }
+        console.log("订阅数据迁移完成", subscribe.length);
+        // 迁移value
+        interface MV2Value {
+          id: number;
+          scriptId: number;
+          storageName?: string;
+          key: string;
+          value: any;
+          createtime: number;
+          updatetime: number;
+        }
+        const values = await db.table("value").toArray();
+        const valueDAO = new ValueDAO();
+        const valueMap = new Map<string, Value>();
+        await Promise.all(
+          values.map((v: MV2Value) => {
+            const { scriptId, storageName, key, value, createtime } = v;
+            return db
+              .table("scripts")
+              .where("id")
+              .equals(scriptId)
+              .first((script: Script) => {
+                if (script) {
+                  let data: { [key: string]: any } = {};
+                  if (!valueMap.has(script.uuid)) {
+                    valueMap.set(script.uuid, {
+                      uuid: script.uuid,
+                      storageName: getStorageName(script),
+                      data: data,
+                      createtime,
+                      updatetime: 0,
+                    });
+                  } else {
+                    data = valueMap.get(script.uuid)!.data;
+                  }
+                  data[key] = value;
+                }
+              });
+          })
+        );
+        // 保存到数据库
+        await Promise.all(
+          Array.from(valueMap.keys()).map((uuid) => {
+            const { storageName, data, createtime } = valueMap.get(uuid)!;
+            return valueDAO.save(storageName!, {
+              uuid,
+              storageName,
+              data,
+              createtime,
+              updatetime: 0,
+            });
+          })
+        );
+        console.log("脚本value数据迁移完成", values.length);
+        // 迁移permission
+        const permissions = await db.table("permission").toArray();
+        const permissionDAO = new PermissionDAO();
+        await Promise.all(
+          permissions.map((p: Permission & { scriptId: number }) => {
+            const { scriptId, permission, permissionValue, createtime, updatetime, allow } = p;
+            return db
+              .table("scripts")
+              .where("id")
+              .equals(scriptId)
+              .first((script: Script) => {
+                if (script) {
+                  return permissionDAO.save({
+                    uuid: script.uuid,
+                    permission,
+                    permissionValue,
+                    createtime,
+                    updatetime,
+                    allow,
+                  });
+                }
+              });
+          })
+        );
+        console.log("脚本permission数据迁移完成", permissions.length);
+        // 打开页面，告知数据储存+升级至了mv3，重启一次扩展
+        setTimeout(async () => {
+          const scripts = await scriptDAO.all();
+          console.log("脚本数据迁移完成", scripts.length);
+          if (scripts.length > 0) {
+            chrome.tabs.create({
+              url: "https://docs.scriptcat.org/docs/change/v0.17/",
+            });
+            setTimeout(() => {
+              chrome.runtime.reload();
+            }, 1000);
+          }
+        }, 2000);
+      } catch (e) {
+        console.error("脚本数据迁移失败", e);
+      }
+    }, 200);
+  });
+  return db.open();
 }
 
 export default function migrate() {
@@ -90,7 +283,8 @@ export default function migrate() {
       value: "++id,scriptId,storageName,key,createtime",
     })
     .upgrade((tx) => {
-      tx.table("value")
+      return tx
+        .table("value")
         .toCollection()
         .modify((value) => {
           if (value.namespace) {
@@ -112,5 +306,5 @@ export default function migrate() {
     permission: "++id,scriptId,[scriptId+permission+permissionValue],createtime,updatetime",
   });
   // 使用小峰驼统一命名规范
-  renameField();
+  return renameField();
 }

src/app/service/offscreen/index.ts

@@ -20,6 +20,10 @@ export class OffscreenManager {
 
   private serviceWorker = new ServiceWorkerClient(this.extensionMessage);
 
+  constructor(private extensionMessage:MessageSend) {
+
+  }
+
   logger(data: Logger) {
     const dao = new LoggerDAO();
     dao.save(data);

src/app/service/service_worker/index.ts

@@ -35,7 +35,15 @@ export default class ServiceWorkerManager {
     const value = new ValueService(this.api.group("value"), this.sender);
     const script = new ScriptService(systemConfig, this.api.group("script"), this.mq, value, resource);
     script.init();
-    const runtime = new RuntimeService(systemConfig, this.api.group("runtime"), this.sender, this.mq, value, script);
+    const runtime = new RuntimeService(
+      systemConfig,
+      this.api.group("runtime"),
+      this.sender,
+      this.mq,
+      value,
+      script,
+      resource
+    );
     runtime.init();
     const popup = new PopupService(this.api.group("popup"), this.mq, runtime);
     popup.init();

src/app/service/service_worker/resource.ts

@@ -36,12 +36,22 @@ export class ResourceService {
     return Promise.resolve(undefined);
   }
 
+  cache: Map<string, { [key: string]: Resource }> = new Map();
+
   public async getScriptResources(script: Script): Promise<{ [key: string]: Resource }> {
-    return Promise.resolve({
+    // 优先从内存中获取
+    if (this.cache.has(script.uuid)) {
+      return Promise.resolve(this.cache.get(script.uuid) || {});
+    }
+    // 资源不存在,重新加载
+    const res = await Promise.resolve({
       ...((await this.getResourceByType(script, "require")) || {}),
       ...((await this.getResourceByType(script, "require-css")) || {}),
       ...((await this.getResourceByType(script, "resource")) || {}),
     });
+    // 缓存到内存
+    this.cache.set(script.uuid, res);
+    return res;
   }
 
   async getResourceByType(script: Script, type: ResourceType): Promise<{ [key: string]: Resource }> {
@@ -159,6 +169,8 @@ export class ResourceService {
   }
 
   public async addResource(url: string, uuid: string, type: ResourceType): Promise<Resource> {
+    // 删除缓存
+    this.cache.delete(uuid);
     const u = this.parseUrl(url);
     let result = await this.getResourceModel(u.url);
     // 资源不存在,重新加载

src/app/service/service_worker/runtime.ts

@@ -26,8 +26,9 @@ import Logger from "@App/app/logger/logger";
 import LoggerCore from "@App/app/logger/core";
 import PermissionVerify from "./permission_verify";
 import { SystemConfig } from "@App/pkg/config/config";
+import { ResourceService } from "./resource";
 
-// 为了优化性能，存储到缓存时删除了code与value
+// 为了优化性能，存储到缓存时删除了code、value与resource
 export interface ScriptMatchInfo extends ScriptRunResouce {
   matches: string[];
   excludeMatches: string[];
@@ -54,7 +55,8 @@ export class RuntimeService {
     private sender: MessageSend,
     private mq: MessageQueue,
     private value: ValueService,
-    private script: ScriptService
+    private script: ScriptService,
+    private resource: ResourceService
   ) {}
 
   async init() {
@@ -212,18 +214,18 @@ export class RuntimeService {
       return scriptRes;
     });
 
-    const enableScript = scripts.filter((item) => item);
+    const enableScript = scripts.filter((item) => item) as ScriptMatchInfo[];
 
     await Promise.all([
       // 加载value
       ...enableScript.map(async (script) => {
         const value = await this.value.getScriptValue(script!);
-        script!.value = value;
+        script.value = value;
       }),
       // 加载resource
       ...enableScript.map(async (script) => {
-        // const resource = await this.script.buildScriptRunResource(script!);
-        // script!.resource = resource;
+        const resource = await this.resource.getScriptResources(script);
+        script.resource = resource;
       }),
     ]);
 
@@ -328,8 +330,10 @@ export class RuntimeService {
     this.scriptMatchCache.forEach((val, key) => {
       scriptMatch[key] = val;
       // 优化性能，将不需要的信息去掉
+      // 而且可能会超过缓存的存储限制
       scriptMatch[key].code = "";
       scriptMatch[key].value = {};
+      scriptMatch[key].resource = {};
     });
     return await Cache.getInstance().set("scriptMatch", scriptMatch);
   }

src/offscreen.ts

@@ -1,21 +1,19 @@
-import migrate from "./app/migrate";
+import { MessageSend } from "@Packages/message/server";
 import LoggerCore from "./app/logger/core";
-import DBWriter from "./app/logger/db_writer";
-import { LoggerDAO } from "./app/repo/logger";
+import MessageWriter from "./app/logger/message_writer";
 import { OffscreenManager } from "./app/service/offscreen";
-
-// 初始化数据库
-migrate();
+import { ExtensionMessageSend } from "@Packages/message/extension_message";
 
 function main() {
   // 初始化日志组件
+  const extensionMessage: MessageSend = new ExtensionMessageSend();
   const loggerCore = new LoggerCore({
-    writer: new DBWriter(new LoggerDAO()),
+    writer: new MessageWriter(extensionMessage),
     labels: { env: "offscreen" },
   });
   loggerCore.logger().debug("offscreen start");
   // 初始化管理器
-  const manager = new OffscreenManager();
+  const manager = new OffscreenManager(extensionMessage);
   manager.initManager();
 }
 

src/pages/confirm/main.tsx

@@ -8,15 +8,12 @@ import "@App/index.css";
 import { Provider } from "react-redux";
 import { store } from "@App/pages/store/store.ts";
 import LoggerCore from "@App/app/logger/core.ts";
-import migrate from "@App/app/migrate.ts";
-import { LoggerDAO } from "@App/app/repo/logger.ts";
-import DBWriter from "@App/app/logger/db_writer.ts";
+import MessageWriter from "@App/app/logger/message_writer.ts";
+import { message } from "../store/global.ts";
 
-// 初始化数据库
-migrate();
 // 初始化日志组件
 const loggerCore = new LoggerCore({
-  writer: new DBWriter(new LoggerDAO()),
+  writer: new MessageWriter(message),
   labels: { env: "confirm" },
 });