Implement cached schema fetching

Fetches the latest schema from the API on game selection. Uses the 304
response of the API to determine if the remote version has been updated
since the last request. Not a huge difference in performance locally but
more efficient on the server side.

Author: ethangreen-dev

src/pages/GameSelectionScreen.vue

@@ -96,6 +96,7 @@ import GameSelectionList from '../components/game-selection/GameSelectionList.vu
 import Game from '../model/game/Game';
 import { capitalize } from '../utils/StringUtils';
 import { StorePlatform as platformLabels } from '../model/platform/StorePlatform';
+import { updateLatestEcosystemSchema } from '../r2mm/ecosystem/EcosystemSchema';
 
 
 const gameSelection = useGameSelectionComposable();
@@ -157,6 +158,9 @@ function selectPlatform() {
 onMounted(async () => {
     window.app.checkForApplicationUpdates();
     await initialize();
+    void updateLatestEcosystemSchema().catch((error) => {
+        console.error('Failed to update latest ecosystem schema.', error);
+    });
 });
 </script>
 

src/r2mm/ecosystem/EcosystemSchema.ts

@@ -12,37 +12,26 @@ import ManagerInformation from "../../_managerinf/ManagerInformation";
 import {EcosystemModloaderPackages, EcosystemSupportedGames} from "../../model/schema/ThunderstoreSchema";
 import {updateModLoaderExports} from "../installing/profile_installers/ModLoaderVariantRecord";
 import LoggerProvider, {LogSeverity} from "../../providers/ror2/logging/LoggerProvider";
+import {getAxiosWithTimeouts} from "../../utils/HttpUtils";
+import {retry} from "../../utils/Common";
 
-export type VersionedThunderstoreEcosystem = ThunderstoreEcosystem & {version: string};
+export type VersionedThunderstoreEcosystem = ThunderstoreEcosystem & {
+    version: string;
+    lastModified?: string;
+};
 
-async function getMergedEcosystemPath(): Promise<string> {
-    return path.join(PathResolver.ROOT, "latest-ecosystem-schema.json");
-}
-
-export async function updateLatestEcosystemSchema(): Promise<void> {
-    const latestSchema = await fetchLatestSchema();
-    await writeLatestEcosystemSchema(latestSchema);
-    await internalUpdateEcosystemReactives(latestSchema);
-}
+type LatestSchemaFetchResult =
+    | {kind: "not-modified"}
+    | {kind: "fetched", schema: ThunderstoreEcosystem, lastModified?: string}
+    | {kind: "failed"};
 
-async function writeLatestEcosystemSchema(schema: ThunderstoreEcosystem): Promise<void> {
-    const asMergedSchema: VersionedThunderstoreEcosystem = {
-        ...schema,
-        version: ManagerInformation.VERSION.toString(),
-    };
-    const writable = JSON.stringify(asMergedSchema);
-    return FsProvider.instance.writeFile(await getMergedEcosystemPath(), writable);
-}
+const ECOSYSTEM_DATA_URL = "https://thunderstore.io/api/experimental/schema/dev/latest/";
 
-async function getLastSavedEcosystemSchema(): Promise<VersionedThunderstoreEcosystem> {
-    const contentBuffer = await FsProvider.instance.readFile(await getMergedEcosystemPath());
-    const content = contentBuffer.toString("utf8");
-    const parsedContent = JSON.parse(content);
-    await validateSchema(parsedContent);
-    return parsedContent;
+async function getMergedEcosystemPath(): Promise<string> {
+    return path.join(PathResolver.ROOT, "latest-ecosystem-schema.json");
 }
 
-async function validateSchema(schema: any): Promise<void> {
+function validateSchema(schema: unknown): ThunderstoreEcosystem {
     const ajv = new Ajv();
     addFormats(ajv);
 
@@ -52,15 +41,15 @@ async function validateSchema(schema: any): Promise<void> {
     if (!isOk) {
         throw new R2Error("Schema validation error", ajv.errorsText(validate.errors));
     }
+
+    return schema as ThunderstoreEcosystem;
 }
 
-async function loadBundledSchema(): Promise<ThunderstoreEcosystem> {
-    await validateSchema(bundledEcosystem);
-    return bundledEcosystem as ThunderstoreEcosystem;
+function loadBundledSchema(): ThunderstoreEcosystem {
+    return validateSchema(bundledEcosystem);
 }
 
-async function fetchLatestSchema(): Promise<ThunderstoreEcosystem> {
-    // TODO - Implement fetching of latest resource
+function createEmptySchema(): ThunderstoreEcosystem {
     return {
         schemaVersion: "",
         communities: {},
@@ -70,16 +59,130 @@ async function fetchLatestSchema(): Promise<ThunderstoreEcosystem> {
     };
 }
 
-async function resolveCachedEcosystemSchema(): Promise<VersionedThunderstoreEcosystem> {
+function mergeSchemas(
+    bundledSchema: ThunderstoreEcosystem,
+    latestSchema: ThunderstoreEcosystem
+): ThunderstoreEcosystem {
+    const modloaderMap = new Map(
+        [...bundledSchema.modloaderPackages, ...latestSchema.modloaderPackages]
+            .map(pkg => [pkg.packageId, pkg])
+    );
+
+    return {
+        schemaVersion: latestSchema.schemaVersion,
+        communities: {
+            ...bundledSchema.communities,
+            ...latestSchema.communities,
+        },
+        games: {
+            ...bundledSchema.games,
+            ...latestSchema.games,
+        },
+        modloaderPackages: [...modloaderMap.values()],
+        packageInstallers: {
+            ...bundledSchema.packageInstallers,
+            ...latestSchema.packageInstallers,
+        },
+    };
+}
+
+async function fetchLatestSchema(
+    currentSchema: VersionedThunderstoreEcosystem | null
+): Promise<LatestSchemaFetchResult> {
+    const timeout = 5000;
+    const requestConfig = {
+        validateStatus: (status: number) => {
+            if (status === 304) {
+                return true;
+            }
+            return status >= 200 && status < 300;
+        },
+        ...(currentSchema?.lastModified ? {headers: {"If-Modified-Since": currentSchema.lastModified}} : {}),
+    };
+
+    try {
+        const axios = getAxiosWithTimeouts(timeout, timeout * 2);
+        const response = await retry(
+            () => axios.get(ECOSYSTEM_DATA_URL, requestConfig),
+            {attempts: 3, interval: 1000, throwLastErrorAsIs: true}
+        );
+        const lastModified = typeof response.headers["last-modified"] === "string"
+            ? response.headers["last-modified"]
+            : undefined;
+
+        if (response.status === 304) {
+            return {kind: "not-modified"};
+        }
+
+        return {
+            kind: "fetched",
+            schema: validateSchema(response.data),
+            ...(lastModified ? {lastModified} : {}),
+        };
+    } catch (e) {
+        console.error(e);
+        return {kind: "failed"};
+    }
+}
+
+export async function updateLatestEcosystemSchema(): Promise<void> {
+    const bundledSchema = loadBundledSchema();
+    const currentSchema = await loadSavedEcosystemSchema();
+    const result = await fetchLatestSchema(currentSchema);
+
+    if (result.kind === "not-modified") {
+        return;
+    }
+
+    if (result.kind === "failed") {
+        if (currentSchema != null) {
+            return;
+        }
+
+        await writeLatestEcosystemSchema(bundledSchema);
+        await internalUpdateEcosystemReactives(bundledSchema);
+        return;
+    }
+
+    const mergedSchema = mergeSchemas(bundledSchema, result.schema);
+    await writeLatestEcosystemSchema(mergedSchema, result.lastModified);
+    await internalUpdateEcosystemReactives(mergedSchema);
+}
+
+async function writeLatestEcosystemSchema(
+    schema: ThunderstoreEcosystem,
+    lastModified?: string
+): Promise<void> {
+    const asMergedSchema: VersionedThunderstoreEcosystem = {
+        ...schema,
+        version: ManagerInformation.VERSION.toString(),
+        ...(lastModified != null ? {lastModified} : {}),
+    };
+    const writable = JSON.stringify(asMergedSchema);
+    return FsProvider.instance.writeFile(await getMergedEcosystemPath(), writable);
+}
+
+async function readSavedEcosystemSchema(): Promise<VersionedThunderstoreEcosystem> {
+    const contentBuffer = await FsProvider.instance.readFile(await getMergedEcosystemPath());
+    const content = contentBuffer.toString("utf8");
+    const parsedContent = JSON.parse(content);
+    const {version, lastModified, ...schemaContent} = parsedContent as VersionedThunderstoreEcosystem;
+    void version;
+    void lastModified;
+    validateSchema(schemaContent);
+    return parsedContent as VersionedThunderstoreEcosystem;
+}
+
+async function loadSavedEcosystemSchema(): Promise<VersionedThunderstoreEcosystem | null> {
     const mergeFilePath = await getMergedEcosystemPath();
-    const bundledSchema = async () => ({...(await loadBundledSchema()), version: ManagerInformation.VERSION.toString()});
     if (!(await FsProvider.instance.exists(mergeFilePath))) {
-        return bundledSchema();
+        return null;
     }
+
     try {
-        let content = await getLastSavedEcosystemSchema();
+        const content = await readSavedEcosystemSchema();
         if (!new VersionNumber(content.version).isEqualTo(ManagerInformation.VERSION)) {
-            return bundledSchema();
+            return null;
         }
         return content;
     } catch (e) {
@@ -88,8 +191,16 @@ async function resolveCachedEcosystemSchema(): Promise<VersionedThunderstoreEcos
             LogSeverity.ERROR,
             `Failed to load cached ecosystem schema, falling back to bundled schema\n${err.message}`
         );
-        return bundledSchema();
+        return null;
+    }
+}
+
+async function resolveCachedEcosystemSchema(): Promise<ThunderstoreEcosystem> {
+    const content = await loadSavedEcosystemSchema();
+    if (content != null) {
+        return content;
     }
+    return loadBundledSchema();
 }
 
 async function internalUpdateEcosystemReactives(schema: ThunderstoreEcosystem): Promise<void> {