fix: transformation errors and add logs

Author: mbani01

services/apps/snowflake_connectors/src/core/transformerBase.ts

@@ -36,7 +36,17 @@ export abstract class TransformerBase {
     try {
       return this.transformRow(row)
     } catch (err) {
-      log.warn({ err, platform: this.platform }, 'Failed to transform row, skipping')
+      const message = err instanceof Error ? err.message : String(err)
+      const stack = err instanceof Error ? err.stack : undefined
+      log.warn(
+        {
+          errMessage: message,
+          errStack: stack,
+          platform: this.platform,
+          rowKeys: Object.keys(row),
+        },
+        'Failed to transform row, skipping',
+      )
       return null
     }
   }

services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts

@@ -41,7 +41,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => {
     org.domain_aliases AS ORG_DOMAIN_ALIASES,
     org.logo_url AS LOGO_URL,
     org.industry AS ORGANIZATION_INDUSTRY,
-    org.n_employees AS ORGANIZATION_SIZE,
+    CAST(org.n_employees AS VARCHAR) AS ORGANIZATION_SIZE,
     ${LFID_COALESCE} AS LFID
   FROM analytics.silver_fact.certificates c
   INNER JOIN cdp_matched_segments cms
@@ -57,7 +57,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => {
   WHERE c.user_email IS NOT NULL`
 
   if (!IS_PROD_ENV) {
-    select += ` AND cms.slug = 'pytorch'`
+    select += ` AND cms.slug = 'cncf'`
   }
 
   const dedup = `

services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts

@@ -48,7 +48,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => {
     org.domain_aliases AS ORG_DOMAIN_ALIASES,
     org.logo_url AS LOGO_URL,
     org.industry AS ORGANIZATION_INDUSTRY,
-    org.n_employees AS ORGANIZATION_SIZE
+    CAST(org.n_employees AS VARCHAR) AS ORGANIZATION_SIZE
   FROM analytics.bronze_census_ti.course_actions ca
   INNER JOIN analytics.bronze_census_ti.users tu
     ON ca.internal_ti_user_id = tu.internal_ti_user_id
@@ -68,7 +68,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => {
     AND tu.user_email IS NOT NULL`
 
   if (!IS_PROD_ENV) {
-    select += ` AND e.project_slug = 'pytorch'`
+    select += ` AND e.project_slug = 'openssf'`
   }
 
   const dedup = `

services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts

@@ -17,12 +17,16 @@ export class TncCoursesTransformer extends TncTransformerBase {
   transformRow(row: Record<string, unknown>): TransformedActivity | null {
     const email = (row.USER_EMAIL as string | null)?.trim() || null
     if (!email) {
-      log.debug({ courseActionId: row.COURSE_ACTION_ID }, 'Skipping row: missing email')
+      log.warn(
+        { courseActionId: row.COURSE_ACTION_ID, rawUserEmail: row.USER_EMAIL },
+        'Skipping row: missing email',
+      )
       return null
     }
 
     const courseActionId = (row.COURSE_ACTION_ID as string | null)?.trim() || null
     if (!courseActionId) {
+      log.warn('Skipping row: missing courseActionId')
       return null
     }
 
@@ -103,6 +107,16 @@ export class TncCoursesTransformer extends TncTransformerBase {
     const segmentSlug = (row.PROJECT_SLUG as string | null)?.trim() || null
     const segmentSourceId = (row.PROJECT_ID as string | null)?.trim() || null
     if (!segmentSlug || !segmentSourceId) {
+      log.warn(
+        {
+          courseActionId,
+          segmentSlug,
+          segmentSourceId,
+          rawProjectSlug: row.PROJECT_SLUG,
+          rawProjectId: row.PROJECT_ID,
+        },
+        'Skipping row: missing segment slug or sourceId',
+      )
       return null
     }
 

services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts

@@ -57,7 +57,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => {
     org.domain_aliases AS ORG_DOMAIN_ALIASES,
     org.logo_url AS LOGO_URL,
     org.industry AS ORGANIZATION_INDUSTRY,
-    org.n_employees AS ORGANIZATION_SIZE,
+    CAST(org.n_employees AS VARCHAR) AS ORGANIZATION_SIZE,
     ${LFID_COALESCE} AS LFID,
     cs.COURSE_STARTED_DATE,
     cs.COURSE_COMPLETED_DATE,
@@ -80,7 +80,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => {
   WHERE e.user_email IS NOT NULL`
 
   if (!IS_PROD_ENV) {
-    select += ` AND e.project_slug = 'pytorch'`
+    select += ` AND e.project_slug = 'openssf'`
   }
 
   const dedup = `

services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts

@@ -17,7 +17,10 @@ export class TncEnrollmentsTransformer extends TncTransformerBase {
   transformRow(row: Record<string, unknown>): TransformedActivity | null {
     const email = (row.USER_EMAIL as string | null)?.trim() || null
     if (!email) {
-      log.debug({ enrollmentId: row.ENROLLMENT_ID }, 'Skipping row: missing email')
+      log.warn(
+        { enrollmentId: row.ENROLLMENT_ID, rawUserEmail: row.USER_EMAIL },
+        'Skipping row: missing email',
+      )
       return null
     }
 
@@ -74,7 +77,7 @@ export class TncEnrollmentsTransformer extends TncTransformerBase {
     } else if (productType?.toLowerCase() === 'training') {
       type = TncActivityType.ENROLLED_TRAINING
     } else {
-      log.debug(
+      log.warn(
         { enrollmentId, productType, instructionType },
         'Skipping row: unrecognized product/instruction type',
       )
@@ -118,6 +121,16 @@ export class TncEnrollmentsTransformer extends TncTransformerBase {
     const segmentSlug = (row.PROJECT_SLUG as string | null)?.trim() || null
     const segmentSourceId = (row.PROJECT_ID as string | null)?.trim() || null
     if (!segmentSlug || !segmentSourceId) {
+      log.warn(
+        {
+          enrollmentId,
+          segmentSlug,
+          segmentSourceId,
+          rawProjectSlug: row.PROJECT_SLUG,
+          rawProjectId: row.PROJECT_ID,
+        },
+        'Skipping row: missing segment slug or sourceId',
+      )
       return null
     }
 

services/apps/snowflake_connectors/src/integrations/tnc/tncTransformerBase.ts

@@ -52,7 +52,10 @@ export abstract class TncTransformerBase extends TransformerBase {
         source: OrganizationSource.TNC,
         identities,
         logo: (row.LOGO_URL as string | null)?.trim() || undefined,
-        size: (row.ORGANIZATION_SIZE as string | null)?.trim() || undefined,
+        size:
+          typeof row.ORGANIZATION_SIZE === 'string'
+            ? row.ORGANIZATION_SIZE.trim() || undefined
+            : undefined,
         industry: (row.ORGANIZATION_INDUSTRY as string | null)?.trim() || undefined,
       },
     ]