fix: lowercase email-shaped identity values (CM-1349) (#4412)

Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com>

Author: skwowet

backend/src/api/public/v1/members/createMember.ts

@@ -54,10 +54,6 @@ export async function createMember(req: Request, res: Response): Promise<void> {
         identities.map((identity) => ({
           ...identity,
           memberId: dbMember.id,
-          value:
-            identity.type === MemberIdentityType.EMAIL
-              ? identity.value.trim().toLowerCase()
-              : identity.value.trim(),
         })),
         true,
         true,

backend/src/api/public/v1/members/identities/createMemberIdentity.ts

@@ -46,10 +46,6 @@ export async function createMemberIdentity(req: Request, res: Response): Promise
     throw new NotFoundError('Member not found')
   }
 
-  // Normalize emails to lowercase; keep username preferred casing from the caller.
-  const normalizedValue =
-    data.type === MemberIdentityType.EMAIL ? data.value.trim().toLowerCase() : data.value.trim()
-
   let result!: IMemberIdentity
   let alreadyExisted = false
 
@@ -59,7 +55,7 @@ export async function createMemberIdentity(req: Request, res: Response): Promise
       captureOldState({})
 
       await qx.tx(async (tx) => {
-        const existing = await findMemberIdentitiesByValue(tx, memberId, normalizedValue, {
+        const existing = await findMemberIdentitiesByValue(tx, memberId, data.value, {
           type: data.type,
         })
         const exactMatch = existing.find((i) => i.platform === data.platform)
@@ -75,7 +71,7 @@ export async function createMemberIdentity(req: Request, res: Response): Promise
                 {
                   memberId,
                   platform: data.platform,
-                  value: normalizedValue,
+                  value: data.value,
                   type: data.type,
                   source: data.source,
                   verified: data.verified,
@@ -105,7 +101,7 @@ export async function createMemberIdentity(req: Request, res: Response): Promise
             }
           }
         } catch (error) {
-          const ctx = { platform: data.platform, value: normalizedValue, type: data.type }
+          const ctx = { platform: data.platform, value: data.value, type: data.type }
           rethrowDbConflict(error, ctx)
         }
 

backend/src/database/repositories/memberRepository.ts

@@ -14,6 +14,7 @@ import {
   Error409,
   RawQueryParser,
   groupBy,
+  normalizeMemberIdentityValue,
 } from '@crowd/common'
 import { BotDetectionService, CommonMemberService } from '@crowd/common_services'
 import {
@@ -1852,6 +1853,7 @@ class MemberRepository {
     const transaction = SequelizeRepository.getTransaction(options)
 
     const seq = SequelizeRepository.getSequelize(options)
+    const normalizedValue = normalizeMemberIdentityValue(value)
 
     const query = `
       insert into "memberIdentities"("memberId", platform, type, value, "tenantId", verified)
@@ -1863,7 +1865,7 @@ class MemberRepository {
       await seq.query(query, {
         replacements: {
           memberId,
-          value,
+          value: normalizedValue,
           type,
           platform,
           tenantId: DEFAULT_TENANT_ID,

backend/src/services/member/memberIdentityService.ts

@@ -13,19 +13,14 @@ import {
   updateMemberIdentity,
 } from '@crowd/data-access-layer/src/members'
 import { LoggerBase } from '@crowd/logging'
-import { IMemberIdentity, MemberIdentityType, NewMemberIdentity } from '@crowd/types'
+import { IMemberIdentity, NewMemberIdentity } from '@crowd/types'
 
 import { IRepositoryOptions } from '@/database/repositories/IRepositoryOptions'
 import SequelizeRepository from '@/database/repositories/sequelizeRepository'
 import { optionsQx } from '@/database/sequelizeQueryExecutor'
 
 import { IServiceOptions } from '../IServiceOptions'
 
-function normalizeIdentityValue(type: string, value: string): string {
-  const trimmed = value.trim()
-  return type === MemberIdentityType.EMAIL ? trimmed.toLowerCase() : trimmed
-}
-
 export default class MemberIdentityService extends LoggerBase {
   options: IServiceOptions
 
@@ -63,16 +58,11 @@ export default class MemberIdentityService extends LoggerBase {
 
           const qx = SequelizeRepository.getQueryExecutor(repoOptions)
 
-          const identityData = {
-            ...data,
-            value: normalizeIdentityValue(data.type, data.value),
-          }
-
           // Check if identity already exists
           const conflict = await findMemberIdentityConflict(qx, {
-            value: identityData.value,
-            platform: identityData.platform,
-            type: identityData.type,
+            value: data.value,
+            platform: data.platform,
+            type: data.type,
           })
 
           if (conflict) {
@@ -87,7 +77,7 @@ export default class MemberIdentityService extends LoggerBase {
           }
 
           // Create member identity
-          await insertMemberIdentities(qx, [{ ...identityData, memberId }])
+          await insertMemberIdentities(qx, [{ ...data, memberId }])
 
           await touchMemberUpdatedAt(qx, memberId)
 
@@ -141,12 +131,7 @@ export default class MemberIdentityService extends LoggerBase {
           const qx = SequelizeRepository.getQueryExecutor(repoOptions)
 
           // Check if any of the identities already exist
-          const normalizedData = data.map((identity) => ({
-            ...identity,
-            value: normalizeIdentityValue(identity.type, identity.value),
-          }))
-
-          for (const identity of normalizedData) {
+          for (const identity of data) {
             const conflict = await findMemberIdentityConflict(qx, {
               value: identity.value,
               platform: identity.platform,
@@ -168,7 +153,7 @@ export default class MemberIdentityService extends LoggerBase {
           // Create member identities
           await insertMemberIdentities(
             qx,
-            normalizedData.map((identity) => ({ ...identity, memberId })),
+            data.map((identity) => ({ ...identity, memberId })),
           )
 
           await touchMemberUpdatedAt(qx, memberId)
@@ -226,10 +211,7 @@ export default class MemberIdentityService extends LoggerBase {
             throw new Error404(this.options.language, 'errors.notFound.message')
           }
 
-          const value = normalizeIdentityValue(
-            data.type ?? currentIdentity.type,
-            data.value ?? currentIdentity.value,
-          )
+          const value = data.value ?? currentIdentity.value
           const platform = data.platform ?? currentIdentity.platform
           const type = data.type ?? currentIdentity.type
 

docs/adr/0015-how-cdp-stores-member-identities.md

@@ -3,7 +3,6 @@
 **Date**: 2026-07-28
 **Status**: accepted
 **Deciders**: Yeganathan S
-**Related**: CM-1349
 
 ## Context
 
@@ -20,18 +19,18 @@ Historically, `memberIdentities` uniqueness was defined on raw `value` (case-sen
 
 Lookups used `lower(value)`, but inserts often did not. Data-sink `mergeData` matched with exact `value ===`, so a self-serve lowercase `willsonhg` plus a later GitHub ingest of `WillsonHG` produced two rows for the same identity — often both verified on the same member. Prod had tens of thousands of GitHub case-variant groups.
 
-Ticket CM-1349 proposed soft-deleting / auto-verifying case variants at verification time. That treats a write-path bug as a product special case.
+Soft-deleting or auto-verifying case variants only at verification time would treat a write-path bug as a product special case.
 
 ## Decision
 
 **Mental model**
 
-| Kind | Store | Compare / unique on |
-| --- | --- | --- |
-| username | preferred casing from the source/integration | `lower(value)` |
-| email | always lowercase | `lower(value)` (same as stored) |
+| Kind     | Store                            | Compare / unique on             |
+| -------- | -------------------------------- | ------------------------------- |
+| username | preferred casing from the source | `lower(value)`                  |
+| email    | always lowercase                 | `lower(value)` (same as stored) |
 
-Identity equality in CDP is `(platform, type, lower(value))`. The `value` column keeps what the source sent for usernames; we do not rewrite GitHub `login` casing on ingest.
+Identity equality in CDP is `(platform, type, lower(value))`. Email vs username for storage casing is inferred from the value with `isValidEmail`, not from `type` (git often stores emails as `type=username`). Non-email usernames keep preferred casing from the source.
 
 **Enforcement**
 
@@ -42,7 +41,7 @@ Identity equality in CDP is `(platform, type, lower(value))`. The `value` column
 
 ## Alternatives Considered
 
-### Alternative 1: Soft-delete / auto-verify case variants at identity verification time (CM-1349 as written)
+### Alternative 1: Soft-delete / auto-verify case variants at identity verification time
 
 - **Pros**: Fixes the user-visible self-serve pain quickly; no schema change.
 - **Cons**: Case variants keep being inserted by ingest/enrichment; verify path becomes a mop; duplicates still break uniqueness and analytics.

services/apps/data_sink_worker/src/service/activity.service.ts

@@ -11,6 +11,8 @@ import {
   generateUUIDv1,
   isDomainExcluded,
   isValidEmail,
+  normalizeMemberIdentities,
+  normalizeMemberIdentityValue,
   parseGitHubNoreplyEmail,
   single,
   singleOrDefault,
@@ -287,7 +289,16 @@ export default class ActivityService extends LoggerBase {
       }
 
       let member = activity.member
-      const username = activity.username ? activity.username.trim() : undefined
+      if (member?.identities) {
+        member.identities = normalizeMemberIdentities(member.identities)
+      }
+
+      const username = activity.username
+        ? normalizeMemberIdentityValue(activity.username)
+        : undefined
+      if (username) {
+        activity.username = username
+      }
       if (!member && username) {
         member = {
           identities: [
@@ -317,17 +328,19 @@ export default class ActivityService extends LoggerBase {
         )
         if (platformIdentity && platformIdentity.value !== username) {
           this.log.debug(
-            { platform, originalUsername: username, correctedUsername: platformIdentity.value },
+            {
+              platform,
+              originalUsername: username,
+              correctedUsername: platformIdentity.value,
+            },
             'Overriding activity.username with member platform identity value',
           )
           activity.username = platformIdentity.value
         }
       }
 
-      member.identities = member.identities.filter((i) => i.value)
-
-      if (!username) {
-        const identities = activity.member.identities.filter(
+      if (!activity.username) {
+        const identities = (member?.identities ?? []).filter(
           (i) => i.platform === platform && i.type === MemberIdentityType.USERNAME,
         )
 
@@ -337,12 +350,12 @@ export default class ActivityService extends LoggerBase {
           // Fall back to same-platform email identity — handles old gerrit records where
           // only a type:email identity was stored (before the gerrit integration
           // gained the email-as-username fallback).
-          const emailFallback = activity.member.identities.find(
+          const emailFallback = (member?.identities ?? []).find(
             (i) => i.platform === platform && i.type === MemberIdentityType.EMAIL && i.value,
           )
           if (emailFallback && emailFallback.verified) {
             activity.username = emailFallback.value
-            activity.member.identities.push({
+            member.identities.push({
               platform,
               type: MemberIdentityType.USERNAME,
               value: emailFallback.value,
@@ -388,12 +401,15 @@ export default class ActivityService extends LoggerBase {
       }
 
       const objectMemberUsername = activity.objectMemberUsername
-        ? activity.objectMemberUsername.trim()
+        ? normalizeMemberIdentityValue(activity.objectMemberUsername)
         : undefined
+      if (objectMemberUsername) {
+        activity.objectMemberUsername = objectMemberUsername
+      }
       let objectMember = activity.objectMember
 
-      if (objectMember) {
-        objectMember.identities = objectMember.identities.filter((i) => i.value)
+      if (objectMember?.identities) {
+        objectMember.identities = normalizeMemberIdentities(objectMember.identities)
       }
 
       if (objectMember && !objectMemberUsername) {
@@ -443,6 +459,11 @@ export default class ActivityService extends LoggerBase {
         }
       }
 
+      activity.member = member
+      if (objectMember) {
+        activity.objectMember = objectMember
+      }
+
       results.set(resultId, { success: true })
     }
 

services/apps/data_sink_worker/src/service/member.service.ts

@@ -8,9 +8,9 @@ import {
   getEarliestValidDate,
   getProperDisplayName,
   isDomainExcluded,
-  isEmail,
   isObjectEmpty,
   isSameMemberIdentity,
+  normalizeMemberIdentities,
   singleOrDefault,
 } from '@crowd/common'
 import {
@@ -136,15 +136,7 @@ export default class MemberService extends LoggerBase {
     // Deduplicate by (platform, type, lower(value)) — prefer verified=true when the same
     // identity appears with both flags. Prevents per-member unique identity index
     // from firing when a payload contains the same identity twice with different verified values.
-    const seen = new Map<string, IMemberIdentity>()
-    for (const id of identities) {
-      const key = `${id.platform}:${id.type}:${id.value.trim().toLowerCase()}`
-      const existing = seen.get(key)
-      if (!existing || (!existing.verified && id.verified)) {
-        seen.set(key, id)
-      }
-    }
-    const deduped = Array.from(seen.values())
+    const deduped = normalizeMemberIdentities(identities)
 
     try {
       await this.memberRepo.insertIdentities(memberId, integrationId, deduped, true)
@@ -346,8 +338,9 @@ export default class MemberService extends LoggerBase {
             )
           }
 
-          // validate emails
-          data.identities = this.validateEmails(data.identities)
+          data.identities = normalizeMemberIdentities(data.identities, {
+            dropInvalidEmails: true,
+          })
 
           data.displayName = getProperDisplayName(data.displayName)
 
@@ -586,11 +579,9 @@ export default class MemberService extends LoggerBase {
             )
           }
 
-          // prevent empty identity handles
-          data.identities = data.identities.filter((i) => i.value)
-
-          // validate emails
-          data.identities = this.validateEmails(data.identities)
+          data.identities = normalizeMemberIdentities(data.identities, {
+            dropInvalidEmails: true,
+          })
 
           // make sure displayName is proper
           if (data.displayName) {
@@ -952,31 +943,6 @@ export default class MemberService extends LoggerBase {
     }
   }
 
-  private validateEmails(identities: IMemberIdentity[]): IMemberIdentity[] {
-    const toReturn: IMemberIdentity[] = []
-
-    for (const identity of identities) {
-      if (identity.type === MemberIdentityType.EMAIL) {
-        const lowerValue = identity.value.toLowerCase()
-        if (
-          isEmail(identity.value) &&
-          toReturn.find(
-            (i) =>
-              i.type === MemberIdentityType.EMAIL &&
-              i.value === lowerValue &&
-              i.platform === identity.platform,
-          ) === undefined
-        ) {
-          toReturn.push({ ...identity, value: lowerValue })
-        }
-      } else {
-        toReturn.push(identity)
-      }
-    }
-
-    return toReturn
-  }
-
   private mergeData(
     dbMember: IDbMember,
     dbIdentities: IMemberIdentity[],