Skip to content

Commit f1fe080

Browse files
bhollisbhollis
authored
Automigrate settings (#294)
* Automigrate settings Ben's account only, for now * Missed a TODO * Types * OK
1 parent 7de56f5 commit f1fe080

8 files changed

Lines changed: 189 additions & 53 deletions

File tree

‎api/db/settings-queries.test.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ it('can insert settings where none exist before', async () => {
1111
showNewItems: true,
1212
});
1313

14-
const settings = await getSettings(client, bungieMembershipId);
14+
const settings = (await getSettings(client, bungieMembershipId))!.settings;
1515
expect(settings.showNewItems).toBe(true);
1616
});
1717
});
@@ -22,14 +22,14 @@ it('can update settings', async () => {
2222
showNewItems: true,
2323
});
2424

25-
const settings = await getSettings(client, bungieMembershipId);
25+
const settings = (await getSettings(client, bungieMembershipId))!.settings;
2626
expect(settings.showNewItems).toBe(true);
2727

2828
await setSetting(client, bungieMembershipId, {
2929
showNewItems: false,
3030
});
3131

32-
const settings2 = await getSettings(client, bungieMembershipId);
32+
const settings2 = (await getSettings(client, bungieMembershipId))!.settings;
3333
expect(settings2.showNewItems).toBe(false);
3434
});
3535
});
@@ -40,14 +40,14 @@ it('can partially update settings', async () => {
4040
showNewItems: true,
4141
});
4242

43-
const settings = await getSettings(client, bungieMembershipId);
43+
const settings = (await getSettings(client, bungieMembershipId))!.settings;
4444
expect(settings.showNewItems).toBe(true);
4545

4646
await setSetting(client, bungieMembershipId, {
4747
singleCharacter: true,
4848
});
4949

50-
const settings2 = await getSettings(client, bungieMembershipId);
50+
const settings2 = (await getSettings(client, bungieMembershipId))!.settings;
5151
expect(settings2.showNewItems).toBe(true);
5252
});
5353
});

‎api/db/settings-queries.ts‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,23 @@ import { Settings } from '../shapes/settings.js';
77
export async function getSettings(
88
client: ClientBase,
99
bungieMembershipId: number,
10-
): Promise<Partial<Settings>> {
11-
const results = await client.query<{ settings: Settings }>({
10+
): Promise<{ settings: Partial<Settings>; deleted: boolean; lastModifiedAt: number } | undefined> {
11+
const results = await client.query<{
12+
settings: Settings;
13+
deleted_at: Date | null;
14+
last_updated_at: Date;
15+
}>({
1216
name: 'get_settings',
13-
text: 'SELECT settings FROM settings WHERE membership_id = $1 and deleted_at IS NULL',
17+
text: 'SELECT settings, deleted_at, last_updated_at FROM settings WHERE membership_id = $1',
1418
values: [bungieMembershipId],
1519
});
16-
return results.rows.length > 0 ? results.rows[0].settings : {};
20+
return results.rows.length > 0
21+
? {
22+
settings: results.rows[0].settings,
23+
deleted: Boolean(results.rows[0].deleted_at),
24+
lastModifiedAt: results.rows[0].last_updated_at.getTime(),
25+
}
26+
: undefined;
1727
}
1828

1929
/**

‎api/routes/profile.ts‎

Lines changed: 48 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,13 @@ import * as Sentry from '@sentry/node';
22
import { ListToken } from '@stately-cloud/client';
33
import express from 'express';
44
import asyncHandler from 'express-async-handler';
5+
import { readTransaction } from '../db/index.js';
6+
import { getSettings } from '../db/settings-queries.js';
57
import { metrics } from '../metrics/index.js';
68
import { ApiApp } from '../shapes/app.js';
79
import { DestinyVersion } from '../shapes/general.js';
810
import { ProfileResponse } from '../shapes/profile.js';
11+
import { defaultSettings } from '../shapes/settings.js';
912
import { UserInfo } from '../shapes/user.js';
1013
import { getProfile, syncProfile } from '../stately/bulk-queries.js';
1114
import { cannedSearches } from '../stately/searches-queries.js';
@@ -141,10 +144,10 @@ function extractSyncToken(syncTokenParam: string | undefined) {
141144
}
142145

143146
try {
144-
const tokenMap = JSON.parse(syncTokenParam) as { [component: string]: string };
145-
return Object.entries(tokenMap).reduce<{ [component: string]: Buffer }>(
147+
const tokenMap = JSON.parse(syncTokenParam) as { [component: string]: string | number };
148+
return Object.entries(tokenMap).reduce<{ [component: string]: Buffer | number }>(
146149
(acc, [component, token]) => {
147-
acc[component] = Buffer.from(token, 'base64');
150+
acc[component] = typeof token === 'string' ? Buffer.from(token, 'base64') : token;
148151
return acc;
149152
},
150153
{},
@@ -162,40 +165,70 @@ async function statelyProfile(
162165
bungieMembershipId: number,
163166
platformMembershipId: string | undefined,
164167
destinyVersion: DestinyVersion,
165-
incomingSyncTokens?: { [component: string]: Buffer },
168+
incomingSyncTokens?: { [component: string]: Buffer | number },
166169
) {
167170
let response: ProfileResponse = {
168171
sync: Boolean(incomingSyncTokens),
169172
};
170173
const timerPrefix = response.sync ? 'profileSync' : 'profileStately';
171174
const counterPrefix = response.sync ? 'sync' : 'stately';
172175
const syncTokens: { [component: string]: string } = {};
173-
const addSyncToken = (name: string, token: ListToken) => {
176+
const addSyncToken = (
177+
name: string,
178+
token: ListToken | { canSync: boolean; tokenData: number },
179+
) => {
174180
if (token.canSync) {
175-
syncTokens[name] = Buffer.from(token.tokenData).toString('base64');
181+
syncTokens[name] =
182+
token.tokenData instanceof Uint8Array
183+
? Buffer.from(token.tokenData).toString('base64')
184+
: token.tokenData.toString();
176185
}
177186
};
178-
const getSyncToken = (name: string) => {
187+
const getSyncToken = <T extends number | Buffer>(name: string) => {
179188
const tokenData = incomingSyncTokens?.[name];
180189
if (incomingSyncTokens && !tokenData) {
181190
throw new Error(`Missing sync token: ${name}`);
182191
}
183-
return tokenData;
192+
return tokenData as T | undefined;
184193
};
185194

186195
// We'll accumulate promises and await them all at the end
187196
const promises: Promise<void>[] = [];
197+
188198
if (components.includes('settings')) {
189199
// TODO: should settings be stored under profile too?? maybe primary profile ID?
190200
promises.push(
191201
(async () => {
202+
// Load settings from Stately. If they're there, you're done. Otherwise load from Postgres.
192203
const start = new Date();
193-
const tokenData = getSyncToken('settings');
194-
const { settings: storedSettings, token: settingsToken } = tokenData
195-
? await syncSettings(tokenData)
196-
: await querySettings(bungieMembershipId);
197-
response.settings = storedSettings;
198-
addSyncToken('settings', settingsToken);
204+
205+
const statelySettings = await querySettings(bungieMembershipId);
206+
if (!statelySettings.settings) {
207+
const now = Date.now();
208+
const pgSettings = await readTransaction(async (pgClient) =>
209+
getSettings(pgClient, bungieMembershipId),
210+
);
211+
if (pgSettings) {
212+
const tokenData = getSyncToken<number>('s');
213+
if (tokenData === undefined || pgSettings.lastModifiedAt > tokenData) {
214+
response.settings = { ...defaultSettings, ...pgSettings.settings };
215+
}
216+
} else {
217+
response.settings = defaultSettings;
218+
}
219+
addSyncToken('s', { canSync: true, tokenData: pgSettings?.lastModifiedAt ?? now });
220+
} else {
221+
const tokenData = getSyncToken<Buffer>('settings');
222+
const { settings: storedSettings, token: settingsToken } = tokenData
223+
? await syncSettings(tokenData)
224+
: {
225+
settings: statelySettings.settings ?? defaultSettings,
226+
token: statelySettings.token,
227+
};
228+
response.settings = storedSettings;
229+
addSyncToken('settings', settingsToken);
230+
}
231+
199232
metrics.timing(`${timerPrefix}.settings`, start);
200233
})(),
201234
);
@@ -225,7 +258,7 @@ async function statelyProfile(
225258
promises.push(
226259
(async () => {
227260
const start = new Date();
228-
const tokenData = getSyncToken(name);
261+
const tokenData = getSyncToken<Buffer>(name);
229262
const { profile, token } = tokenData
230263
? await syncProfile(tokenData)
231264
: await getProfile(platformMembershipId, destinyVersion, suffix);

‎api/routes/update.ts‎

Lines changed: 70 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import express from 'express';
44
import asyncHandler from 'express-async-handler';
55
import { transaction } from '../db/index.js';
66
import { backfillMigrationState } from '../db/migration-state-queries.js';
7+
import { replaceSettings, setSetting as setSettingInPostgres } from '../db/settings-queries.js';
78
import { metrics } from '../metrics/index.js';
89
import { ApiApp } from '../shapes/app.js';
910
import { DestinyVersion } from '../shapes/general.js';
@@ -25,7 +26,7 @@ import {
2526
UsedSearchUpdate,
2627
} from '../shapes/profile.js';
2728
import { SearchType } from '../shapes/search.js';
28-
import { Settings } from '../shapes/settings.js';
29+
import { defaultSettings, Settings } from '../shapes/settings.js';
2930
import { UserInfo } from '../shapes/user.js';
3031
import { client } from '../stately/client.js';
3132
import {
@@ -42,14 +43,19 @@ import {
4243
UpdateSearch,
4344
updateSearches,
4445
} from '../stately/searches-queries.js';
45-
import { setSetting as setSettingInStately } from '../stately/settings-queries.js';
46+
import {
47+
deleteSettings as deleteSettingsInStately,
48+
getSettingsForUpdate,
49+
setSetting as setSettingInStately,
50+
} from '../stately/settings-queries.js';
4651
import { trackUntrackTriumphs } from '../stately/triumphs-queries.js';
4752
import {
4853
badRequest,
4954
checkPlatformMembershipId,
5055
delay,
5156
isValidItemId,
5257
isValidPlatformMembershipId,
58+
subtractObject,
5359
} from '../utils.js';
5460

5561
/**
@@ -213,14 +219,17 @@ function validateUpdates(
213219
}
214220

215221
switch (update.action) {
216-
case 'setting':
217222
case 'tag_cleanup':
218223
case 'delete_loadout':
219224
case 'track_triumph':
220225
case 'delete_search':
221226
// no special validation
222227
break;
223228

229+
case 'setting':
230+
result = validateUpdateSettings(update.payload);
231+
break;
232+
224233
case 'loadout':
225234
result = validateUpdateLoadout(update.payload);
226235
break;
@@ -303,7 +312,28 @@ async function statelyUpdate(
303312
for (const update of group as SettingUpdate[]) {
304313
mergedSettings = { ...mergedSettings, ...update.payload };
305314
}
306-
await setSettingInStately(txn, bungieMembershipId, mergedSettings);
315+
316+
if (bungieMembershipId === 7094) {
317+
const statelySettings = await getSettingsForUpdate(txn, bungieMembershipId);
318+
319+
if (statelySettings) {
320+
mergedSettings = { ...statelySettings, ...mergedSettings };
321+
await transaction(async (pgClient) => {
322+
await replaceSettings(
323+
pgClient,
324+
bungieMembershipId,
325+
subtractObject(mergedSettings, defaultSettings),
326+
);
327+
});
328+
await deleteSettingsInStately(bungieMembershipId);
329+
} else {
330+
await transaction(async (pgClient) => {
331+
await setSettingInPostgres(pgClient, bungieMembershipId, mergedSettings);
332+
});
333+
}
334+
} else {
335+
await setSettingInStately(txn, bungieMembershipId, mergedSettings);
336+
}
307337
break;
308338
}
309339

@@ -506,6 +536,42 @@ async function statelyUpdate(
506536
// metrics.timing('update.loadout', start);
507537
// }
508538

539+
/** Helper function to validate integer ranges */
540+
function validateIntRange(
541+
value: unknown,
542+
fieldName: string,
543+
min: number,
544+
max: number,
545+
): string | undefined {
546+
if (value === undefined) {
547+
return undefined;
548+
}
549+
if (typeof value !== 'number' || !Number.isInteger(value) || value < min || value > max) {
550+
metrics.increment(`update.validation.${fieldName}OutOfRange.count`);
551+
return `${fieldName} must be an integer between ${min} and ${max}`;
552+
}
553+
return undefined;
554+
}
555+
556+
function validateUpdateSettings(settings: Partial<Settings>): ProfileUpdateResult {
557+
const errors = [
558+
// Validate numeric ranges
559+
validateIntRange(settings.charCol, 'charCol', 2, 5),
560+
validateIntRange(settings.charColMobile, 'charColMobile', 2, 5),
561+
validateIntRange(settings.itemSize, 'itemSize', 0, 66),
562+
validateIntRange(settings.inventoryClearSpaces, 'inventoryClearSpaces', 0, 9),
563+
].filter((e) => e !== undefined);
564+
565+
if (errors.length > 0) {
566+
return {
567+
status: 'InvalidArgument',
568+
message: errors.join('; '),
569+
};
570+
}
571+
572+
return { status: 'Success' };
573+
}
574+
509575
function validateUpdateLoadout(loadout: Loadout): ProfileUpdateResult {
510576
return validateLoadout('update', loadout) ?? { status: 'Success' };
511577
}

‎api/stately/bulk-queries.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ describe('deleteAllDataForUser', () => {
8080
),
8181
).toEqual([]);
8282
expect((await getTrackedTriumphsForProfile(platformMembershipId)).triumphs).toEqual([]);
83-
expect((await getSettings(bungieMembershipId)).showNewItems).toBe(false);
83+
expect((await getSettings(bungieMembershipId))?.showNewItems).toBe(undefined);
8484
});
8585
});
8686

‎api/stately/bulk-queries.ts‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import { captureMessage } from '@sentry/node';
22
import { keyPath, ListToken } from '@stately-cloud/client';
3+
import { readTransaction, transaction } from '../db/index.js';
4+
import { deleteSettings, getSettings as getSettingsFromPostgres } from '../db/settings-queries.js';
35
import { DeleteAllResponse } from '../shapes/delete-all.js';
46
import { ExportResponse } from '../shapes/export.js';
57
import { DestinyVersion } from '../shapes/general.js';
@@ -12,7 +14,7 @@ import { convertItemAnnotation, keyFor as tagKeyFor } from './item-annotations-q
1214
import { convertItemHashTag, keyFor as hashTagKeyFor } from './item-hash-tags-queries.js';
1315
import { convertLoadoutFromStately, keyFor as loadoutKeyFor } from './loadouts-queries.js';
1416
import { convertSearchFromStately, keyFor as searchKeyFor } from './searches-queries.js';
15-
import { deleteSettings, getSettings } from './settings-queries.js';
17+
import { deleteSettings as deleteSettingsInStately, getSettings } from './settings-queries.js';
1618
import { batches, fromStatelyUUID, parseKeyPath } from './stately-utils.js';
1719
import { keyFor as triumphKeyFor } from './triumphs-queries.js';
1820

@@ -26,7 +28,9 @@ export async function deleteAllDataForUser(
2628
const responses = await Promise.all(platformMembershipIds.map((p) => deleteAllDataForProfile(p)));
2729

2830
// Also delete settings, which are stored by membershipId
29-
await deleteSettings(bungieMembershipId);
31+
await deleteSettingsInStately(bungieMembershipId);
32+
// And delete from Postgres too
33+
await transaction(async (pgClient) => deleteSettings(pgClient, bungieMembershipId));
3034

3135
const response = responses.reduce<DeleteAllResponse['deleted']>(
3236
(acc, r) => {
@@ -111,10 +115,16 @@ export async function exportDataForUser(
111115
bungieMembershipId: number,
112116
platformMembershipIds: string[],
113117
): Promise<ExportResponse> {
114-
const settingsPromise = getSettings(bungieMembershipId);
118+
let settings = await getSettings(bungieMembershipId);
119+
if (!settings) {
120+
const pgSettings = await readTransaction((client) =>
121+
getSettingsFromPostgres(client, bungieMembershipId),
122+
);
123+
settings = { ...defaultSettings, ...pgSettings?.settings };
124+
}
125+
115126
const responses = await Promise.all(platformMembershipIds.map((p) => exportDataForProfile(p)));
116127

117-
const settings = await settingsPromise;
118128
const initialResponse: ExportResponse = {
119129
settings: subtractObject(settings, defaultSettings),
120130
loadouts: [],

0 commit comments

Comments
 (0)