Skip to content

Commit aa7ee43

Browse files
bhollisbhollis
authored
Implement the postgres post-migrated paths (#298)
* Handle postgres data in update * Maybe implement profile * Test both backends * Clean up some old queries * Implement import, export, and delete-my-data * Handle separate test import * Fixes * Compile error * Forgot the migration * Don't need this test anymore
1 parent c818e7d commit aa7ee43

28 files changed

Lines changed: 2884 additions & 1923 deletions

‎.vscode/settings.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,5 +104,6 @@
104104
],
105105
"yaml.schemas": {
106106
"https://json.schemastore.org/github-workflow.json": "file:///Users/brh/Documents/oss/dim-api/.github/workflows/deploy.yml"
107-
}
107+
},
108+
"js/ts.tsdk.path": "node_modules/typescript/lib"
108109
}

‎api/db/global-settings-queries.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ export async function setGlobalSettings(flavor: string, settings: Partial<Global
1313
return pool.query({
1414
name: 'set_global_settings',
1515
text: `
16-
INSERT INTO global_settings (flavor, settings, updated_at)
17-
VALUES ($1, $2, NOW())
16+
INSERT INTO global_settings (flavor, settings)
17+
VALUES ($1, $2)
1818
ON CONFLICT (flavor)
1919
DO UPDATE SET settings = (global_settings.settings || $2)
2020
`,

‎api/db/item-annotations-queries.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import {
44
deleteItemAnnotation,
55
deleteItemAnnotationList,
66
getItemAnnotationsForProfile,
7+
softDeleteAllItemAnnotations,
78
updateItemAnnotation,
89
} from './item-annotations-queries.js';
910

@@ -132,3 +133,41 @@ it('can clear tags', async () => {
132133
expect(annotations).toEqual([]);
133134
});
134135
});
136+
137+
it('can soft delete all annotations and recreate them', async () => {
138+
await transaction(async (client) => {
139+
await updateItemAnnotation(client, bungieMembershipId, platformMembershipId, 2, {
140+
id: '123456',
141+
tag: 'favorite',
142+
notes: 'the best',
143+
});
144+
145+
// Verify it exists
146+
let annotations = await getItemAnnotationsForProfile(client, platformMembershipId, 2);
147+
expect(annotations).toEqual([
148+
{
149+
id: '123456',
150+
tag: 'favorite',
151+
notes: 'the best',
152+
},
153+
]);
154+
155+
await softDeleteAllItemAnnotations(client, platformMembershipId, 2);
156+
157+
annotations = await getItemAnnotationsForProfile(client, platformMembershipId, 2);
158+
expect(annotations).toEqual([]);
159+
160+
await updateItemAnnotation(client, bungieMembershipId, platformMembershipId, 2, {
161+
id: '123456',
162+
tag: 'junk',
163+
});
164+
165+
annotations = await getItemAnnotationsForProfile(client, platformMembershipId, 2);
166+
expect(annotations).toEqual([
167+
{
168+
id: '123456',
169+
tag: 'junk',
170+
},
171+
]);
172+
});
173+
});

‎api/db/item-annotations-queries.ts‎

Lines changed: 75 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { partition } from 'es-toolkit';
12
import { ClientBase, QueryResult } from 'pg';
23
import { metrics } from '../metrics/index.js';
34
import { DestinyVersion } from '../shapes/general.js';
@@ -7,6 +8,7 @@ interface ItemAnnotationRow {
78
inventory_item_id: string;
89
tag: TagValue | null;
910
notes: string | null;
11+
deleted_at: Date | null;
1012
crafted_date: Date | null;
1113
}
1214

@@ -37,31 +39,26 @@ export async function getItemAnnotationsForProfile(
3739
}
3840

3941
/**
40-
* Get ALL of the item annotations for a particular user across all platforms.
42+
* Get all of the item annotations for a particular platform_membership_id and destiny_version that have changed since the token timestamp, including all tombstones.
4143
*/
42-
export async function getAllItemAnnotationsForUser(
44+
export async function syncItemAnnotationsForProfile(
4345
client: ClientBase,
44-
bungieMembershipId: number,
45-
): Promise<
46-
{
47-
platformMembershipId: string;
48-
destinyVersion: DestinyVersion;
49-
annotation: ItemAnnotation;
50-
}[]
51-
> {
52-
// TODO: this isn't indexed!
53-
const results = await client.query<
54-
ItemAnnotationRow & { platform_membership_id: string; destiny_version: DestinyVersion }
55-
>({
56-
name: 'get_all_item_annotations',
57-
text: 'SELECT platform_membership_id, destiny_version, inventory_item_id, tag, notes, crafted_date FROM item_annotations WHERE inventory_item_id != 0 and platform_membership_id = $1 and deleted_at IS NULL',
58-
values: [bungieMembershipId],
46+
platformMembershipId: string,
47+
destinyVersion: DestinyVersion,
48+
syncTimestamp: number,
49+
): Promise<{ updated: ItemAnnotation[]; deletedItemIds: string[] }> {
50+
const results = await client.query<ItemAnnotationRow>({
51+
name: 'sync_item_annotations',
52+
text: 'SELECT inventory_item_id, tag, notes, crafted_date, deleted_at FROM item_annotations WHERE platform_membership_id = $1 and destiny_version = $2 and last_updated_at > $3',
53+
values: [platformMembershipId, destinyVersion, new Date(syncTimestamp)],
5954
});
60-
return results.rows.map((row) => ({
61-
platformMembershipId: row.platform_membership_id,
62-
destinyVersion: row.destiny_version,
63-
annotation: convertItemAnnotation(row),
64-
}));
55+
56+
const [updatedRows, deletedRows] = partition(results.rows, (row) => row.deleted_at === null);
57+
58+
return {
59+
updated: updatedRows.map(convertItemAnnotation),
60+
deletedItemIds: deletedRows.map((row) => row.inventory_item_id),
61+
};
6562
}
6663

6764
function convertItemAnnotation(row: ItemAnnotationRow): ItemAnnotation {
@@ -98,10 +95,45 @@ export async function updateItemAnnotation(
9895
}
9996
const response = await client.query({
10097
name: 'upsert_item_annotation',
101-
text: `insert INTO item_annotations (membership_id, platform_membership_id, destiny_version, inventory_item_id, tag, notes, crafted_date)
102-
values ($1, $2, $3, $4, (CASE WHEN $5 = 0 THEN NULL ELSE $5 END), (CASE WHEN $6 = 'clear' THEN NULL ELSE $6 END), $7)
103-
on conflict (platform_membership_id, inventory_item_id)
104-
do update set (tag, notes, crafted_date, deleted_at) = ((CASE WHEN $5 = 0 THEN NULL WHEN $5 IS NULL THEN item_annotations.tag ELSE $5 END), (CASE WHEN $6 = 'clear' THEN NULL WHEN $6 IS NULL THEN item_annotations.notes ELSE $6 END), $7, null)`,
98+
text: `
99+
INSERT INTO item_annotations (
100+
membership_id,
101+
platform_membership_id,
102+
destiny_version,
103+
inventory_item_id,
104+
tag,
105+
notes,
106+
crafted_date
107+
)
108+
VALUES (
109+
$1,
110+
$2,
111+
$3,
112+
$4,
113+
(CASE WHEN $5 = 0 THEN NULL ELSE $5 END),
114+
(CASE WHEN $6 = 'clear' THEN NULL ELSE $6 END),
115+
$7
116+
)
117+
ON CONFLICT (platform_membership_id, inventory_item_id)
118+
DO UPDATE SET
119+
tag = (CASE
120+
WHEN $5 = 0 THEN NULL
121+
WHEN $5 IS NULL THEN (CASE WHEN item_annotations.deleted_at IS NULL THEN item_annotations.tag ELSE NULL END)
122+
ELSE $5
123+
END),
124+
notes = (CASE
125+
WHEN $6 = 'clear' THEN NULL
126+
WHEN $6 IS NULL THEN (CASE WHEN item_annotations.deleted_at IS NULL THEN item_annotations.notes ELSE NULL END)
127+
ELSE $6
128+
END),
129+
crafted_date = $7,
130+
deleted_at = (CASE
131+
WHEN (CASE WHEN $5 = 0 THEN NULL WHEN $5 IS NULL THEN (CASE WHEN item_annotations.deleted_at IS NULL THEN item_annotations.tag ELSE NULL END) ELSE $5 END) IS NULL
132+
AND (CASE WHEN $6 = 'clear' THEN NULL WHEN $6 IS NULL THEN (CASE WHEN item_annotations.deleted_at IS NULL THEN item_annotations.notes ELSE NULL END) ELSE $6 END) IS NULL
133+
THEN now()
134+
ELSE NULL
135+
END)
136+
`,
105137
values: [
106138
bungieMembershipId, // $1
107139
platformMembershipId, // $2
@@ -147,7 +179,7 @@ export async function deleteItemAnnotation(
147179
): Promise<QueryResult> {
148180
return client.query({
149181
name: 'delete_item_annotation',
150-
text: `update item_annotations set (tag, notes, deleted_at) = (null, null, now()) where platform_membership_id = $1 and inventory_item_id = $2`,
182+
text: `update item_annotations set deleted_at = now() where platform_membership_id = $1 and inventory_item_id = $2 and deleted_at is null`,
151183
values: [platformMembershipId, inventoryItemId],
152184
});
153185
}
@@ -162,7 +194,7 @@ export async function deleteItemAnnotationList(
162194
): Promise<QueryResult> {
163195
return client.query({
164196
name: 'delete_item_annotation_list',
165-
text: `update item_annotations set (tag, notes, deleted_at) = (null, null, now()) where platform_membership_id = $1 and inventory_item_id::bigint = ANY($2::bigint[])`,
197+
text: `update item_annotations set deleted_at = now() where platform_membership_id = $1 and inventory_item_id::bigint = ANY($2::bigint[]) and deleted_at is null`,
166198
values: [platformMembershipId, inventoryItemIds],
167199
});
168200
}
@@ -181,3 +213,18 @@ export async function deleteAllItemAnnotations(
181213
values: [bungieMembershipId],
182214
});
183215
}
216+
217+
/**
218+
* Soft-delete all item annotations for a platform (sets deleted_at timestamp for sync support).
219+
*/
220+
export async function softDeleteAllItemAnnotations(
221+
client: ClientBase,
222+
platformMembershipId: string,
223+
destinyVersion: DestinyVersion,
224+
): Promise<QueryResult> {
225+
return client.query({
226+
name: 'soft_delete_all_item_annotations',
227+
text: `update item_annotations set deleted_at = now() where platform_membership_id = $1 and destiny_version = $2 and deleted_at is null`,
228+
values: [platformMembershipId, destinyVersion],
229+
});
230+
}

‎api/db/item-hash-tags-queries.test.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {
33
deleteAllItemHashTags,
44
deleteItemHashTag,
55
getItemHashTagsForProfile,
6+
softDeleteAllItemHashTags,
67
updateItemHashTag,
78
} from './item-hash-tags-queries.js';
89

@@ -111,3 +112,39 @@ it('can delete item hash tags by setting both values to null/empty', async () =>
111112
expect(annotations).toEqual([]);
112113
});
113114
});
115+
116+
it('handles soft delete properly', async () => {
117+
await transaction(async (client) => {
118+
// Create a hash tag
119+
await updateItemHashTag(client, bungieMembershipId, platformMembershipId, {
120+
hash: 2926662838,
121+
tag: 'favorite',
122+
notes: 'the best',
123+
});
124+
125+
let annotations = await getItemHashTagsForProfile(client, platformMembershipId);
126+
expect(annotations).toHaveLength(1);
127+
expect(annotations[0]).toEqual({
128+
hash: 2926662838,
129+
tag: 'favorite',
130+
notes: 'the best',
131+
});
132+
133+
await softDeleteAllItemHashTags(client, platformMembershipId);
134+
135+
annotations = await getItemHashTagsForProfile(client, platformMembershipId);
136+
expect(annotations).toEqual([]);
137+
138+
await updateItemHashTag(client, bungieMembershipId, platformMembershipId, {
139+
hash: 2926662838,
140+
tag: 'keep',
141+
});
142+
143+
annotations = await getItemHashTagsForProfile(client, platformMembershipId);
144+
expect(annotations).toHaveLength(1);
145+
expect(annotations[0]).toEqual({
146+
hash: 2926662838,
147+
tag: 'keep',
148+
});
149+
});
150+
});

‎api/db/item-hash-tags-queries.ts‎

Lines changed: 75 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { partition } from 'es-toolkit';
12
import { ClientBase, QueryResult } from 'pg';
23
import { metrics } from '../metrics/index.js';
34
import { ItemHashTag, TagValue } from '../shapes/item-annotations.js';
@@ -7,23 +8,46 @@ interface ItemHashTagRow {
78
item_hash: string;
89
tag: TagValue | null;
910
notes: string | null;
11+
deleted_at: Date | null;
1012
}
1113

1214
/**
13-
* Get all of the hash tags for a particular platform_membership_id and destiny_version.
15+
* Get all of the hash tags for a particular platform_membership_id.
1416
*/
1517
export async function getItemHashTagsForProfile(
1618
client: ClientBase,
1719
platformMembershipId: string,
1820
): Promise<ItemHashTag[]> {
19-
const results = await client.query({
21+
const results = await client.query<ItemHashTagRow>({
2022
name: 'get_item_hash_tags',
2123
text: 'SELECT item_hash, tag, notes FROM item_hash_tags WHERE platform_membership_id = $1 and deleted_at IS NULL',
2224
values: [platformMembershipId],
2325
});
2426
return results.rows.map(convertItemHashTag);
2527
}
2628

29+
/**
30+
* Get all of the hash tags for a particular platform_membership_id that have changed since syncTimestamp, including tombstones.
31+
*/
32+
export async function syncItemHashTagsForProfile(
33+
client: ClientBase,
34+
platformMembershipId: string,
35+
syncTimestamp: number,
36+
): Promise<{ updated: ItemHashTag[]; deletedItemHashes: number[] }> {
37+
const results = await client.query<ItemHashTagRow>({
38+
name: 'sync_item_hash_tags',
39+
text: 'SELECT item_hash, tag, notes, deleted_at FROM item_hash_tags WHERE platform_membership_id = $1 and last_updated_at > $2',
40+
values: [platformMembershipId, new Date(syncTimestamp)],
41+
});
42+
43+
const [updatedRows, deletedRows] = partition(results.rows, (row) => row.deleted_at === null);
44+
45+
return {
46+
updated: updatedRows.map(convertItemHashTag),
47+
deletedItemHashes: deletedRows.map((row) => parseInt(row.item_hash, 10)),
48+
};
49+
}
50+
2751
function convertItemHashTag(row: ItemHashTagRow): ItemHashTag {
2852
const result: ItemHashTag = {
2953
hash: parseInt(row.item_hash, 10),
@@ -55,10 +79,40 @@ export async function updateItemHashTag(
5579

5680
const response = await client.query({
5781
name: 'upsert_hash_tag',
58-
text: `insert INTO item_hash_tags (membership_id, platform_membership_id, item_hash, tag, notes)
59-
values ($1, $2, $3, (CASE WHEN $4 = 0 THEN NULL ELSE $4 END), (CASE WHEN $5 = 'clear' THEN NULL ELSE $5 END))
60-
on conflict (platform_membership_id, item_hash)
61-
do update set (tag, notes, deleted_at) = ((CASE WHEN $4 = 0 THEN NULL WHEN $4 IS NULL THEN item_hash_tags.tag ELSE $4 END), (CASE WHEN $5 = 'clear' THEN NULL WHEN $5 IS NULL THEN item_hash_tags.notes ELSE $5 END), null)`,
82+
text: `
83+
INSERT INTO item_hash_tags (
84+
membership_id,
85+
platform_membership_id,
86+
item_hash,
87+
tag,
88+
notes
89+
)
90+
VALUES (
91+
$1,
92+
$2,
93+
$3,
94+
(CASE WHEN $4 = 0 THEN NULL ELSE $4 END),
95+
(CASE WHEN $5 = 'clear' THEN NULL ELSE $5 END)
96+
)
97+
ON CONFLICT (platform_membership_id, item_hash)
98+
DO UPDATE SET
99+
tag = (CASE
100+
WHEN $4 = 0 THEN NULL
101+
WHEN $4 IS NULL THEN (CASE WHEN item_hash_tags.deleted_at IS NULL THEN item_hash_tags.tag ELSE NULL END)
102+
ELSE $4
103+
END),
104+
notes = (CASE
105+
WHEN $5 = 'clear' THEN NULL
106+
WHEN $5 IS NULL THEN (CASE WHEN item_hash_tags.deleted_at IS NULL THEN item_hash_tags.notes ELSE NULL END)
107+
ELSE $5
108+
END),
109+
deleted_at = (CASE
110+
WHEN (CASE WHEN $4 = 0 THEN NULL WHEN $4 IS NULL THEN (CASE WHEN item_hash_tags.deleted_at IS NULL THEN item_hash_tags.tag ELSE NULL END) ELSE $4 END) IS NULL
111+
AND (CASE WHEN $5 = 'clear' THEN NULL WHEN $5 IS NULL THEN (CASE WHEN item_hash_tags.deleted_at IS NULL THEN item_hash_tags.notes ELSE NULL END) ELSE $5 END) IS NULL
112+
THEN now()
113+
ELSE NULL
114+
END)
115+
`,
62116
values: [
63117
bungieMembershipId,
64118
platformMembershipId,
@@ -102,7 +156,7 @@ export async function deleteItemHashTag(
102156
): Promise<QueryResult> {
103157
return client.query({
104158
name: 'delete_item_hash_tag',
105-
text: `update item_hash_tags set (tag, notes, deleted_at) = (null, null, now()) where platform_membership_id = $1 and item_hash = $2`,
159+
text: `update item_hash_tags set deleted_at = now() where platform_membership_id = $1 and item_hash = $2 and deleted_at is null`,
106160
values: [platformMembershipId, itemHash],
107161
});
108162
}
@@ -120,3 +174,17 @@ export async function deleteAllItemHashTags(
120174
values: [platformMembershipId],
121175
});
122176
}
177+
178+
/**
179+
* Soft-delete all item hash tags for a platform (sets deleted_at timestamp for sync support).
180+
*/
181+
export async function softDeleteAllItemHashTags(
182+
client: ClientBase,
183+
platformMembershipId: string,
184+
): Promise<QueryResult> {
185+
return client.query({
186+
name: 'soft_delete_all_item_hash_tags',
187+
text: `update item_hash_tags set deleted_at = now() where platform_membership_id = $1 and deleted_at is null`,
188+
values: [platformMembershipId],
189+
});
190+
}

0 commit comments

Comments
 (0)