Skip to content

Commit bce2a4c

Browse files
skwowetskwowet
committed
feat: add script to cleanup lowercase email-shaped identities
Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com>
1 parent 3f81bd5 commit bce2a4c

3 files changed

Lines changed: 395 additions & 94 deletions

File tree

‎backend/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
"script:fix-members-activities-after-unaffilation": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/fix-members-activities-after-unaffilation.ts",
3535
"script:process-bot-members": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/process-bot-members.ts",
3636
"script:cleanup-same-member-case-variant-identities": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/cleanup-same-member-case-variant-identities.ts",
37+
"script:cleanup-lowercase-email-shaped-identities": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/cleanup-lowercase-email-shaped-identities.ts",
3738
"script:backfill-email-domain-member-organization-dates": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/backfill-email-domain-member-organization-dates.ts",
3839
"script:onboard-default-tenant": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/onboard-default-tenant.ts",
3940
"script:onboard-default-tenant:local": "set -a && . ./.env.dist.local && . ./.env.override.local && set +a && pnpm run script:onboard-default-tenant",

‎backend/src/bin/scripts/cleanup-lowercase-email-shaped-identities.ts‎

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
/**
2+
* Lowercase email-shaped member identity values (and matching activityRelations usernames).
3+
*
4+
* Scans live identities where value <> lower(value) and value contains '@'.
5+
* Uses normalizeMemberIdentityValue (isValidEmail) — non-email-shaped values are skipped.
6+
*
7+
* Usage:
8+
* pnpm run script:cleanup-lowercase-email-shaped-identities
9+
* pnpm run script:cleanup-lowercase-email-shaped-identities -- --testRun
10+
*/
11+
import commandLineArgs from 'command-line-args'
12+
13+
import { normalizeMemberIdentityValue } from '@crowd/common'
14+
import { pgpQx } from '@crowd/data-access-layer'
15+
import { getDbConnection } from '@crowd/data-access-layer/src/database'
16+
import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor'
17+
import { getServiceLogger } from '@crowd/logging'
18+
19+
import { DB_CONFIG } from '@/conf'
20+
21+
const log = getServiceLogger()
22+
23+
const options = [
24+
{
25+
name: 'testRun',
26+
alias: 't',
27+
type: Boolean,
28+
},
29+
]
30+
31+
const parameters = commandLineArgs(options)
32+
33+
type MixedCaseEmailIdentity = {
34+
id: string
35+
memberId: string
36+
platform: string
37+
type: string
38+
value: string
39+
}
40+
41+
const AR_UPDATE_BATCH_SIZE = 5000
42+
const LOAD_BATCH_SIZE = 500
43+
44+
async function findMixedCaseEmailIdentities(
45+
qx: QueryExecutor,
46+
afterId: string | null,
47+
limit: number,
48+
): Promise<MixedCaseEmailIdentity[]> {
49+
return qx.select(
50+
`
51+
select id, "memberId", platform, type, value
52+
from "memberIdentities"
53+
where "deletedAt" is null
54+
and value <> lower(value)
55+
and position('@' in value) > 0
56+
${afterId ? 'and id > $(afterId)' : ''}
57+
order by id
58+
limit $(limit)
59+
`,
60+
{ afterId, limit },
61+
)
62+
}
63+
64+
async function lowercaseIdentityValue(qx: QueryExecutor, id: string, value: string): Promise<number> {
65+
return qx.result(
66+
`
67+
update "memberIdentities"
68+
set
69+
value = $(value),
70+
"updatedAt" = now()
71+
where id = $(id)
72+
and "deletedAt" is null
73+
and value <> $(value)
74+
`,
75+
{ id, value },
76+
)
77+
}
78+
79+
async function rewriteActivityRelationUsernamesToLower(
80+
qx: QueryExecutor,
81+
memberId: string,
82+
platform: string,
83+
lowerValue: string,
84+
): Promise<{ usernameRows: number; objectMemberUsernameRows: number }> {
85+
let usernameRows = 0
86+
let objectMemberUsernameRows = 0
87+
let updated: number
88+
89+
do {
90+
updated = await qx.result(
91+
`
92+
update "activityRelations"
93+
set
94+
username = $(lowerValue),
95+
"updatedAt" = now()
96+
where "activityId" in (
97+
select "activityId"
98+
from "activityRelations"
99+
where "memberId" = $(memberId)
100+
and platform = $(platform)
101+
and lower(username) = $(lowerValue)
102+
and username <> $(lowerValue)
103+
limit $(batchSize)
104+
)
105+
`,
106+
{
107+
memberId,
108+
platform,
109+
lowerValue,
110+
batchSize: AR_UPDATE_BATCH_SIZE,
111+
},
112+
)
113+
usernameRows += updated
114+
} while (updated === AR_UPDATE_BATCH_SIZE)
115+
116+
do {
117+
updated = await qx.result(
118+
`
119+
update "activityRelations"
120+
set
121+
"objectMemberUsername" = $(lowerValue),
122+
"updatedAt" = now()
123+
where "activityId" in (
124+
select "activityId"
125+
from "activityRelations"
126+
where "objectMemberId" = $(memberId)
127+
and platform = $(platform)
128+
and lower("objectMemberUsername") = $(lowerValue)
129+
and "objectMemberUsername" <> $(lowerValue)
130+
limit $(batchSize)
131+
)
132+
`,
133+
{
134+
memberId,
135+
platform,
136+
lowerValue,
137+
batchSize: AR_UPDATE_BATCH_SIZE,
138+
},
139+
)
140+
objectMemberUsernameRows += updated
141+
} while (updated === AR_UPDATE_BATCH_SIZE)
142+
143+
return { usernameRows, objectMemberUsernameRows }
144+
}
145+
146+
setImmediate(async () => {
147+
const testRun = parameters.testRun ?? false
148+
const PROCESS_BATCH_LOG_EVERY = testRun ? 1 : 200
149+
const batchSize = testRun ? 10 : LOAD_BATCH_SIZE
150+
151+
const db = await getDbConnection({
152+
host: DB_CONFIG.writeHost,
153+
port: DB_CONFIG.port,
154+
database: DB_CONFIG.database,
155+
user: DB_CONFIG.username,
156+
password: DB_CONFIG.password,
157+
})
158+
159+
const qx = pgpQx(db)
160+
161+
log.info({ testRun, batchSize }, 'Lowercasing email-shaped identity values!')
162+
163+
let afterId: string | null = null
164+
let processed = 0
165+
let identitiesUpdated = 0
166+
let skipped = 0
167+
let arUsernameUpdated = 0
168+
let arObjectUsernameUpdated = 0
169+
170+
for (;;) {
171+
const candidates = await findMixedCaseEmailIdentities(qx, afterId, batchSize)
172+
if (candidates.length === 0) {
173+
break
174+
}
175+
176+
afterId = candidates[candidates.length - 1].id
177+
178+
for (const row of candidates) {
179+
const normalized = normalizeMemberIdentityValue(row.value)
180+
181+
if (normalized !== row.value) {
182+
if (testRun) {
183+
log.info(
184+
{
185+
memberId: row.memberId,
186+
platform: row.platform,
187+
type: row.type,
188+
from: row.value,
189+
to: normalized,
190+
},
191+
'Lowercasing email-shaped identity!',
192+
)
193+
}
194+
195+
const { updatedCount, ar } = await qx.tx(async (tx) => {
196+
const updatedCount = await lowercaseIdentityValue(tx, row.id, normalized)
197+
const ar = await rewriteActivityRelationUsernamesToLower(
198+
tx,
199+
row.memberId,
200+
row.platform,
201+
normalized,
202+
)
203+
return { updatedCount, ar }
204+
})
205+
206+
identitiesUpdated += updatedCount
207+
arUsernameUpdated += ar.usernameRows
208+
arObjectUsernameUpdated += ar.objectMemberUsernameRows
209+
processed += 1
210+
211+
if (processed % PROCESS_BATCH_LOG_EVERY === 0) {
212+
log.info(
213+
{
214+
processed,
215+
identitiesUpdated,
216+
skipped,
217+
arUsernameUpdated,
218+
arObjectUsernameUpdated,
219+
},
220+
'Progress!',
221+
)
222+
}
223+
} else {
224+
skipped += 1
225+
}
226+
}
227+
228+
if (testRun) {
229+
log.info('Test run - stopping after first batch!')
230+
break
231+
}
232+
}
233+
234+
log.info(
235+
{
236+
processed,
237+
identitiesUpdated,
238+
skipped,
239+
arUsernameUpdated,
240+
arObjectUsernameUpdated,
241+
},
242+
'Done!',
243+
)
244+
245+
process.exit(0)
246+
})

0 commit comments

Comments
 (0)