🔀 Merge pull request #753 from FrostCo/feat/data-migration-improvements

Feat/data migration improvements

Author: richardfrost

src/script/DataMigration.ts

@@ -1,5 +1,5 @@
 import Constants from '@APF/lib/Constants';
-import { booleanToNumber, getVersion, isVersionOlder } from '@APF/lib/helper';
+import { booleanToNumber, deepCloneJson, getVersion, isVersionOlder } from '@APF/lib/helper';
 import WebConfig from '@APF/WebConfig';
 import type { WordOptions } from '@APF/lib/Word';
 
@@ -13,6 +13,17 @@ export interface Migration {
 export default class DataMigration {
   cfg: WebConfig;
 
+  /**
+   * Which high-level runner is executing migration methods: `'import'` for {@link DataMigration.runImportMigrations}
+   * or `'versionUpgrade'` for {@link DataMigration.byVersion}. Otherwise `null`.
+   */
+  private _migrationRunner: 'import' | 'versionUpgrade' | null = null;
+
+  /** `true` only while `runImportMigrations` is running (parsed JSON / in-memory config), not during extension upgrade. */
+  get isConfigImportMigration(): boolean {
+    return this._migrationRunner === 'import';
+  }
+
   //#region Class reference helpers
   // Can be overridden in children classes
   static get Config() {
@@ -73,6 +84,116 @@ export default class DataMigration {
     this.cfg = config;
   }
 
+  /**
+   * Merges host maps for {@link DataMigration.loadFormerLargeKeyStorage}: later `part` wins per host and per
+   * field within a host.
+   */
+  static mergeFormerLargeKeyParts(
+    acc: Record<string, unknown> | null,
+    part: Record<string, unknown> | null,
+  ): Record<string, unknown> | null {
+    if (part == null || !Object.keys(part).length) return acc;
+    if (acc == null || !Object.keys(acc).length) return deepCloneJson(part) as Record<string, unknown>;
+    const out = deepCloneJson(acc) as Record<string, unknown>;
+    for (const h of Object.keys(part)) {
+      const O = part[h];
+      const A = out[h];
+      if (A && typeof A === 'object' && !Array.isArray(A) && O && typeof O === 'object' && !Array.isArray(O)) {
+        out[h] = { ...(A as object), ...(O as object) };
+      } else {
+        out[h] = deepCloneJson(O);
+      }
+    }
+    return out;
+  }
+
+  /**
+   * Loads a retired large-key blob when it is no longer in {@link WebConfig._largeKeys}. Merges **non-split**
+   * (`formerLogicalKey` as a single object in sync or local) and **split** (`_${key}N` containers) reads;
+   * later sources overlay earlier (bare local → bare sync → local splits → sync splits). Returns `null` if
+   * nothing was stored. Pair with {@link DataMigration.removeFormerLargeKeyStorage}.
+   */
+  static async loadFormerLargeKeyStorage(
+    cfg: WebConfig,
+    formerLogicalKey: string,
+  ): Promise<Record<string, unknown> | null> {
+    const C = this.Config;
+    if (!C.chromeStorageAvailable() || !formerLogicalKey) return null;
+
+    const parseSplitContainers = (raw: Record<string, unknown>): Record<string, unknown> | null => {
+      const temp = { ...raw };
+      const combinedKeys = C.combineData(temp, formerLogicalKey);
+      if (!combinedKeys?.length) return null;
+      const obj = temp[formerLogicalKey];
+      if (!obj || typeof obj !== 'object' || Array.isArray(obj)) return null;
+      if (!Object.keys(obj).length) return null;
+      return deepCloneJson(obj) as Record<string, unknown>;
+    };
+
+    const readBareLocal = async (): Promise<Record<string, unknown> | null> => {
+      const localData = (await C.getLocalStorage([formerLogicalKey])) as Record<string, unknown>;
+      const obj = localData[formerLogicalKey];
+      if (obj == null || typeof obj !== 'object' || Array.isArray(obj)) return null;
+      if (!Object.keys(obj).length) return null;
+      return deepCloneJson(obj) as Record<string, unknown>;
+    };
+
+    const readBareSync = async (): Promise<Record<string, unknown> | null> => {
+      const syncData = (await C.getSyncStorage([formerLogicalKey])) as Record<string, unknown>;
+      const obj = syncData[formerLogicalKey];
+      if (obj == null || typeof obj !== 'object' || Array.isArray(obj)) return null;
+      if (!Object.keys(obj).length) return null;
+      return deepCloneJson(obj) as Record<string, unknown>;
+    };
+
+    const readSyncSplits = async (): Promise<Record<string, unknown> | null> => {
+      const raw = (await C.getSyncStorage(C.splitKeyNames(formerLogicalKey))) as Record<string, unknown>;
+      return parseSplitContainers(raw);
+    };
+
+    const readLocalSplits = async (): Promise<Record<string, unknown> | null> => {
+      const raw = (await C.getLocalStorage(C.splitKeyNames(formerLogicalKey))) as Record<string, unknown>;
+      return parseSplitContainers(raw);
+    };
+
+    const [bareLocal, bareSync, localSplits, syncSplits] = await Promise.all([
+      readBareLocal(),
+      readBareSync(),
+      readLocalSplits(),
+      readSyncSplits(),
+    ]);
+
+    let merged: Record<string, unknown> | null = null;
+    merged = this.mergeFormerLargeKeyParts(merged, bareLocal);
+    merged = this.mergeFormerLargeKeyParts(merged, bareSync);
+    merged = this.mergeFormerLargeKeyParts(merged, localSplits);
+    merged = this.mergeFormerLargeKeyParts(merged, syncSplits);
+
+    return merged;
+  }
+
+  /** @see DataMigration.loadFormerLargeKeyStorage */
+  loadFormerLargeKeyStorage(formerLogicalKey: string): Promise<Record<string, unknown> | null> {
+    return (this.constructor as typeof DataMigration).loadFormerLargeKeyStorage(this.cfg, formerLogicalKey);
+  }
+
+  /**
+   * Removes the bare logical key (non-split) and all split containers (`_${key}N`) from sync **and** local
+   * storage. Use when the key was removed from {@link WebConfig._largeKeys} so {@link WebConfig.remove} would
+   * not expand split names.
+   */
+  static async removeFormerLargeKeyStorage(formerLogicalKey: string): Promise<void> {
+    const C = this.Config;
+    if (!C.chromeStorageAvailable() || !formerLogicalKey) return;
+    const keysToRemove = [formerLogicalKey, ...C.splitKeyNames(formerLogicalKey)];
+    await Promise.all([C.removeSyncStorage(keysToRemove), C.removeLocalStorage(keysToRemove)]);
+  }
+
+  /** @see DataMigration.removeFormerLargeKeyStorage */
+  removeFormerLargeKeyStorage(formerLogicalKey: string): Promise<void> {
+    return (this.constructor as typeof DataMigration).removeFormerLargeKeyStorage(formerLogicalKey);
+  }
+
   // TODO: Only tested with arrays
   _renameConfigKeys(oldCfg: WebConfig, oldKeys: string[], mapping: { [key: string]: string }) {
     for (const oldKey of oldKeys) {
@@ -101,12 +222,18 @@ export default class DataMigration {
   async byVersion(oldVersion: string) {
     const version = getVersion(oldVersion);
     let migrated = false;
-    for (const migration of (this.constructor as typeof DataMigration).migrations) {
-      if (isVersionOlder(version, getVersion(migration.version))) {
-        migrated = true;
-        if (migration.async) await this[migration.name]();
-        else this[migration.name]();
+    const prevRunner = this._migrationRunner;
+    this._migrationRunner = 'versionUpgrade';
+    try {
+      for (const migration of (this.constructor as typeof DataMigration).migrations) {
+        if (isVersionOlder(version, getVersion(migration.version))) {
+          migrated = true;
+          if (migration.async) await this[migration.name]();
+          else this[migration.name]();
+        }
       }
+    } finally {
+      this._migrationRunner = prevRunner;
     }
 
     return migrated;
@@ -200,13 +327,18 @@ export default class DataMigration {
 
   async runImportMigrations() {
     let migrated = false;
-
-    for (const migration of (this.constructor as typeof DataMigration).migrations) {
-      if (migration.runOnImport) {
-        migrated = true;
-        if (migration.async) await this[migration.name]();
-        else this[migration.name]();
+    const prevRunner = this._migrationRunner;
+    this._migrationRunner = 'import';
+    try {
+      for (const migration of (this.constructor as typeof DataMigration).migrations) {
+        if (migration.runOnImport) {
+          migrated = true;
+          if (migration.async) await this[migration.name]();
+          else this[migration.name]();
+        }
       }
+    } finally {
+      this._migrationRunner = prevRunner;
     }
 
     return migrated;