Remove Stately paths (#303)

* Postgres only settings, sync

* Remove stately from profile

* Remove Stately from update

* Remove Stately from import/export

* Remove Stately from delete-all-data

* Update tests

Author: bhollis

api/db/settings-queries.ts

@@ -26,6 +26,32 @@ export async function getSettings(
     : undefined;
 }
 
+/**
+ * Get settings for a particular account.
+ */
+export async function syncSettings(
+  client: ClientBase,
+  bungieMembershipId: number,
+  syncTimestamp: number,
+): Promise<{ settings: Partial<Settings>; deleted: boolean; lastModifiedAt: number } | undefined> {
+  const results = await client.query<{
+    settings: Partial<Settings>;
+    deleted_at: Date | null;
+    last_updated_at: Date;
+  }>({
+    name: 'sync_settings',
+    text: 'SELECT settings, deleted_at, last_updated_at FROM settings WHERE membership_id = $1 and last_updated_at > $2',
+    values: [bungieMembershipId, new Date(syncTimestamp)],
+  });
+  return results.rows.length > 0
+    ? {
+        settings: results.rows[0].settings,
+        deleted: Boolean(results.rows[0].deleted_at),
+        lastModifiedAt: results.rows[0].last_updated_at.getTime(),
+      }
+    : undefined;
+}
+
 /**
  * Insert or update (upsert) an entire settings tree, totally replacing whatever's there.
  */

api/routes/delete-all-data.ts

@@ -11,15 +11,21 @@ import { softDeleteAllTrackedTriumphs } from '../db/triumphs-queries.js';
 import { DeleteAllResponse } from '../shapes/delete-all.js';
 import { DestinyVersion } from '../shapes/general.js';
 import { UserInfo } from '../shapes/user.js';
-import { deleteAllDataForUser } from '../stately/bulk-queries.js';
 
 /**
  * Delete My Data - this allows a user to wipe all their data from DIM storage.
  */
 export const deleteAllDataHandler = asyncHandler(async (req, res) => {
   const { bungieMembershipId, profileIds } = req.user as UserInfo;
 
-  let result = await deleteAllDataForUser(bungieMembershipId, profileIds);
+  let result = {
+    settings: 1,
+    loadouts: 0,
+    tags: 0,
+    itemHashTags: 0,
+    triumphs: 0,
+    searches: 0,
+  };
 
   await transaction(async (client) => {
     await deleteSettings(client, bungieMembershipId);

api/routes/export.ts

@@ -4,17 +4,12 @@ import { readTransaction } from '../db/index.js';
 import { getItemAnnotationsForProfile } from '../db/item-annotations-queries.js';
 import { getItemHashTagsForProfile } from '../db/item-hash-tags-queries.js';
 import { getLoadoutsForProfile } from '../db/loadouts-queries.js';
-import { getMigrationState, MigrationState } from '../db/migration-state-queries.js';
 import { getSearchesForProfile } from '../db/searches-queries.js';
 import { getSettings as getSettingsFromPostgres } from '../db/settings-queries.js';
 import { getTrackedTriumphsForProfile } from '../db/triumphs-queries.js';
 import { ExportResponse } from '../shapes/export.js';
 import { DestinyVersion } from '../shapes/general.js';
-import { defaultSettings, Settings } from '../shapes/settings.js';
 import { UserInfo } from '../shapes/user.js';
-import { exportDataForProfile } from '../stately/bulk-queries.js';
-import { getSettings } from '../stately/settings-queries.js';
-import { subtractObject } from '../utils.js';
 
 export const exportHandler = asyncHandler(async (req, res) => {
   const { bungieMembershipId, profileIds } = req.user as UserInfo;
@@ -31,20 +26,11 @@ export const exportHandler = asyncHandler(async (req, res) => {
   };
 
   for (const profileId of profileIds) {
-    const migrationState = await readTransaction(async (client) =>
-      getMigrationState(client, profileId),
-    );
-
-    let partialResponse: ExportResponse;
-    if (migrationState.state === MigrationState.Postgres) {
-      partialResponse = await readTransaction(async (client) => {
-        const d1Response = await pgExport(client, profileId, 1);
-        const d2Response = await pgExport(client, profileId, 2);
-        return mergeResponses(d1Response, d2Response);
-      });
-    } else {
-      partialResponse = await exportDataForProfile(profileId);
-    }
+    const partialResponse = await readTransaction(async (client) => {
+      const d1Response = await pgExport(client, profileId, 1);
+      const d2Response = await pgExport(client, profileId, 2);
+      return mergeResponses(d1Response, d2Response);
+    });
 
     response = mergeResponses(response, partialResponse);
   }
@@ -71,16 +57,10 @@ function mergeResponses(base: ExportResponse, addition: ExportResponse): ExportR
 export async function exportSettings(
   bungieMembershipId: number,
 ): Promise<ExportResponse['settings']> {
-  let settings: Partial<Settings>;
   const pgSettings = await readTransaction((client) =>
     getSettingsFromPostgres(client, bungieMembershipId),
   );
-  if (pgSettings) {
-    settings = pgSettings.settings;
-  } else {
-    settings = subtractObject((await getSettings(bungieMembershipId)) ?? {}, defaultSettings);
-  }
-  return settings;
+  return pgSettings?.settings ?? {};
 }
 
 export async function pgExport(

api/routes/import.ts

@@ -7,7 +7,6 @@ import {
 } from '../db/item-annotations-queries.js';
 import { softDeleteAllItemHashTags, updateItemHashTag } from '../db/item-hash-tags-queries.js';
 import { softDeleteAllLoadouts, updateLoadout } from '../db/loadouts-queries.js';
-import { doMigration, getMigrationState, MigrationState } from '../db/migration-state-queries.js';
 import { importSearch, softDeleteAllSearches } from '../db/searches-queries.js';
 import { replaceSettings } from '../db/settings-queries.js';
 import { softDeleteAllTrackedTriumphs, trackTriumph } from '../db/triumphs-queries.js';
@@ -18,7 +17,6 @@ import { ItemAnnotation, ItemHashTag } from '../shapes/item-annotations.js';
 import { Loadout } from '../shapes/loadouts.js';
 import { defaultSettings, Settings } from '../shapes/settings.js';
 import { UserInfo } from '../shapes/user.js';
-import { deleteAllDataForUser } from '../stately/bulk-queries.js';
 import { badRequest, subtractObject } from '../utils.js';
 
 export const importHandler = asyncHandler(async (req, res) => {
@@ -120,25 +118,7 @@ export const importHandler = asyncHandler(async (req, res) => {
       response.itemHashTags += importResp.itemHashTags;
     };
 
-    const migrationState = await transaction(async (client) =>
-      getMigrationState(client, profileId),
-    );
-
-    if (migrationState.state === MigrationState.MigratingToPostgres) {
-      badRequest(
-        res,
-        `Unable to import data for profile ${profileId} - migration in progress. Please wait a bit and try again.`,
-      );
-      return;
-    }
-
-    if (migrationState.state === MigrationState.Stately) {
-      await doMigration(bungieMembershipId, profileId, doImport, async () =>
-        deleteAllDataForUser(bungieMembershipId, [profileId]),
-      );
-    } else {
-      await doImport();
-    }
+    await doImport();
   }
 
   // default 200 OK

api/routes/profile.ts

@@ -1,5 +1,4 @@
 import * as Sentry from '@sentry/node';
-import { ListToken } from '@stately-cloud/client';
 import express from 'express';
 import asyncHandler from 'express-async-handler';
 import { readTransaction } from '../db/index.js';
@@ -12,9 +11,8 @@ import {
   syncItemHashTagsForProfile,
 } from '../db/item-hash-tags-queries.js';
 import { getLoadoutsForProfile, syncLoadoutsForProfile } from '../db/loadouts-queries.js';
-import { getMigrationState, MigrationState } from '../db/migration-state-queries.js';
 import { getSearchesForProfile, syncSearchesForProfile } from '../db/searches-queries.js';
-import { getSettings } from '../db/settings-queries.js';
+import { getSettings, syncSettings } from '../db/settings-queries.js';
 import {
   getTrackedTriumphsForProfile,
   syncTrackedTriumphsForProfile,
@@ -26,8 +24,6 @@ import { ProfileResponse } from '../shapes/profile.js';
 import { Search, SearchType } from '../shapes/search.js';
 import { defaultSettings } from '../shapes/settings.js';
 import { UserInfo } from '../shapes/user.js';
-import { getProfile, syncProfile } from '../stately/bulk-queries.js';
-import { querySettings, syncSettings } from '../stately/settings-queries.js';
 import { badRequest, checkPlatformMembershipId, isValidPlatformMembershipId } from '../utils.js';
 
 type ProfileComponent = 'settings' | 'loadouts' | 'tags' | 'hashtags' | 'triumphs' | 'searches';
@@ -224,21 +220,15 @@ async function loadProfile(
   destinyVersion: DestinyVersion,
   incomingSyncTokens?: { [component: string]: Buffer | number },
 ) {
-  let response: ProfileResponse = {
+  const response: ProfileResponse = {
     sync: Boolean(incomingSyncTokens),
   };
   const timerPrefix = response.sync ? 'profileSync' : 'profileStately';
   const counterPrefix = response.sync ? 'sync' : 'stately';
-  const syncTokens: { [component: string]: string | number } = {};
-  const addSyncToken = (
-    name: string,
-    token: ListToken | { canSync: boolean; tokenData: number },
-  ) => {
+  const syncTokens: { [component: string]: number } = {};
+  const addSyncToken = (name: string, token: { canSync: boolean; tokenData: number }) => {
     if (token.canSync) {
-      syncTokens[name] =
-        token.tokenData instanceof Uint8Array
-          ? Buffer.from(token.tokenData).toString('base64')
-          : token.tokenData;
+      syncTokens[name] = token.tokenData;
     }
   };
   const getSyncToken = <T extends number | Buffer>(name: string) => {
@@ -256,55 +246,38 @@ async function loadProfile(
     // TODO: should settings be stored under profile too?? maybe primary profile ID?
     promises.push(
       (async () => {
-        // Load settings from Postgres. If they're there, you're done. Otherwise load from Stately.
         const start = new Date();
-
         const now = Date.now();
+        const tokenData = getSyncToken<number>('s');
         // TODO: Should add the token to the query to avoid fetching if unchanged
         const pgSettings = await readTransaction(async (pgClient) =>
-          getSettings(pgClient, bungieMembershipId),
+          tokenData
+            ? syncSettings(pgClient, bungieMembershipId, tokenData)
+            : getSettings(pgClient, bungieMembershipId),
         );
-        if (pgSettings) {
-          const tokenData = getSyncToken<number>('s');
-          if (tokenData === undefined || pgSettings.lastModifiedAt > tokenData) {
-            response.settings = { ...defaultSettings, ...pgSettings.settings };
-          }
-          addSyncToken('s', { canSync: true, tokenData: now });
-        } else {
-          const tokenData = getSyncToken<Buffer>('settings');
-          const { settings: storedSettings, token: settingsToken } = tokenData
-            ? await syncSettings(tokenData)
-            : await querySettings(bungieMembershipId);
-          response.settings = storedSettings;
-          addSyncToken('settings', settingsToken);
+        if (
+          tokenData === undefined ||
+          (pgSettings !== undefined && pgSettings.lastModifiedAt > tokenData)
+        ) {
+          response.settings = { ...defaultSettings, ...pgSettings?.settings };
         }
+        addSyncToken('s', { canSync: true, tokenData: now });
 
         metrics.timing(`${timerPrefix}.settings`, start);
       })(),
     );
   }
 
-  let loadFromPostgres = false;
   if (
-    platformMembershipId &&
     (['loadouts', 'tags', 'hashtags', 'triumphs', 'searches'] as const).some((c) =>
       components.includes(c),
     )
   ) {
-    const { state: migrationState } = await readTransaction(async (client) =>
-      getMigrationState(client, platformMembershipId),
-    );
-
-    if (migrationState === MigrationState.Postgres) {
-      loadFromPostgres = true;
-    }
-  }
-
-  if (loadFromPostgres) {
     if (!platformMembershipId) {
       badRequest(res, `Need a platformMembershipId to return ${components.join(', ')}`);
       return;
     }
+
     promises.push(
       (async () => {
         const now = Date.now();
@@ -446,71 +419,6 @@ async function loadProfile(
         });
       })(),
     );
-  } else {
-    // Special case: DIM wants everything, so we can get it in a single query
-    if (
-      platformMembershipId &&
-      (['loadouts', 'tags', 'hashtags', 'triumphs', 'searches'] as const).every((c) =>
-        components.includes(c),
-      )
-    ) {
-      // Replace the individual components with a bulk fetch
-      components = components.includes('settings') ? ['settings', 'p'] : ['p'];
-    }
-
-    const loadComponent = (
-      name: Exclude<ProfileComponent, 'settings'> | 'p',
-      suffix: string,
-      handleEmpty: () => void,
-    ) => {
-      if (components.includes(name)) {
-        if (!platformMembershipId) {
-          badRequest(res, `Need a platformMembershipId to return ${name}`);
-          return;
-        }
-        promises.push(
-          (async () => {
-            const start = new Date();
-            const tokenData = getSyncToken<Buffer>(name);
-            const { profile, token } = tokenData
-              ? await syncProfile(tokenData)
-              : await getProfile(platformMembershipId, destinyVersion, suffix);
-            response = { ...response, ...profile };
-            if (!tokenData) {
-              handleEmpty();
-            }
-            addSyncToken(name, token);
-            metrics.timing(`${timerPrefix}.${name}`, start);
-          })(),
-        );
-      }
-    };
-
-    loadComponent('p', '', () => {
-      response.loadouts ??= [];
-      response.searches ??= [];
-      response.tags ??= [];
-      response.itemHashTags ??= [];
-      response.triumphs ??= [];
-      response.searches ??= [];
-    });
-    loadComponent('loadouts', '/loadout', () => {
-      response.loadouts ??= [];
-    });
-    loadComponent('tags', '/ia', () => {
-      response.tags ??= [];
-    });
-    if (destinyVersion === 2) {
-      loadComponent('hashtags', '/iht', () => {
-        response.itemHashTags ??= [];
-      });
-    }
-    loadComponent('triumphs', '/triumph', () => {
-      response.triumphs ??= [];
-    });
-    loadComponent('searches', '/search', () => {
-      response.searches ??= [];
-    });
   }
 
   await Promise.all(promises);