Skip to content

Commit 3a96f52

Browse files
mbani01mbani01
committed
feat: finalize queries & transformations
1 parent 4e3cbe0 commit 3a96f52

10 files changed

Lines changed: 329 additions & 130 deletions

File tree

‎services/apps/snowflake_connectors/src/integrations/index.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ import { buildSourceQuery as cventBuildSourceQuery } from './cvent/event-registr
1010
import { CventTransformer } from './cvent/event-registrations/transformer'
1111
import { buildSourceQuery as tncCertificatesBuildQuery } from './tnc/certificates/buildSourceQuery'
1212
import { TncCertificatesTransformer } from './tnc/certificates/transformer'
13-
import { buildSourceQuery as tncCourseActionsBuildQuery } from './tnc/course-actions/buildSourceQuery'
14-
import { TncCourseActionsTransformer } from './tnc/course-actions/transformer'
13+
import { buildSourceQuery as tncCoursesBuildQuery } from './tnc/courses/buildSourceQuery'
14+
import { TncCoursesTransformer } from './tnc/courses/transformer'
1515
import { buildSourceQuery as tncEnrollmentsBuildQuery } from './tnc/enrollments/buildSourceQuery'
1616
import { TncEnrollmentsTransformer } from './tnc/enrollments/transformer'
1717
import { DataSource, DataSourceName, PlatformDefinition } from './types'
@@ -42,9 +42,9 @@ const supported: Partial<Record<PlatformType, PlatformDefinition>> = {
4242
transformer: new TncCertificatesTransformer(),
4343
},
4444
{
45-
name: DataSourceName.TNC_COURSE_ACTIONS,
46-
buildSourceQuery: tncCourseActionsBuildQuery,
47-
transformer: new TncCourseActionsTransformer(),
45+
name: DataSourceName.TNC_COURSES,
46+
buildSourceQuery: tncCoursesBuildQuery,
47+
transformer: new TncCoursesTransformer(),
4848
},
4949
],
5050
},

‎services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
import { IS_PROD_ENV } from '@crowd/common'
22

3+
// Main: analytics.silver_fact.certificates (certificate data)
4+
// Joins:
5+
// - analytics.silver_dim._crowd_dev_segments_union (segment resolution)
6+
// - analytics.bronze_fivetran_salesforce.bronze_salesforce_merged_user (LFID)
7+
// - analytics.silver_dim.users (LFID fallback)
8+
// - analytics.bronze_fivetran_salesforce.accounts + analytics.bronze_fivetran_salesforce_b2b.accounts (org data)
9+
310
const CDP_MATCHED_SEGMENTS = `
411
cdp_matched_segments AS (
512
SELECT DISTINCT
@@ -47,8 +54,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => {
4754
AND u.lf_username IS NOT NULL
4855
LEFT JOIN org_accounts org
4956
ON c.account_id = org.account_id
50-
WHERE c.user_email IS NOT NULL
51-
AND c.is_deleted = false`
57+
WHERE c.user_email IS NOT NULL`
5258

5359
if (!IS_PROD_ENV) {
5460
select += ` AND cms.slug = 'pytorch'`

‎services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,6 @@ export class TncCertificatesTransformer extends TransformerBase {
8888
},
8989
attributes: {
9090
productName: (row.COURSE_NAME as string | null) || null,
91-
productType: 'Certification',
9291
technology: (row.TECHNOLOGIES_LIST as string | null) || null,
9392
didExpire: row.DID_EXPIRE as boolean | null,
9493
expirationDate: (row.EXPIRATION_DATE as string | null) || null,

‎services/apps/snowflake_connectors/src/integrations/tnc/course-actions/buildSourceQuery.ts‎

Lines changed: 0 additions & 50 deletions
This file was deleted.

‎services/apps/snowflake_connectors/src/integrations/tnc/course-actions/transformer.ts‎

Lines changed: 0 additions & 68 deletions
This file was deleted.

‎services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
import { IS_PROD_ENV } from '@crowd/common'
2+
3+
// Main: analytics.bronze_census_ti.course_actions (course action data)
4+
// Joins:
5+
// - analytics.bronze_census_ti.users (user resolution via internal_ti_user_id)
6+
// - analytics.bronze_census_ti.courses (course metadata)
7+
// - analytics.silver_fact.enrollments (segment + org resolution via email + course_id)
8+
// - analytics.silver_dim._crowd_dev_segments_union (segment resolution)
9+
// - analytics.bronze_fivetran_salesforce.accounts + analytics.bronze_fivetran_salesforce_b2b.accounts (org data)
10+
11+
const CDP_MATCHED_SEGMENTS = `
12+
cdp_matched_segments AS (
13+
SELECT DISTINCT
14+
s.SOURCE_ID AS sourceId,
15+
s.slug
16+
FROM ANALYTICS.SILVER_DIM._CROWD_DEV_SEGMENTS_UNION s
17+
WHERE s.PARENT_SLUG IS NOT NULL
18+
AND s.GRANDPARENTS_SLUG IS NOT NULL
19+
AND s.SOURCE_ID IS NOT NULL
20+
)`
21+
22+
const ORG_ACCOUNTS = `
23+
org_accounts AS (
24+
SELECT account_id, account_name, website, domain_aliases, LOGO_URL, INDUSTRY, N_EMPLOYEES
25+
FROM analytics.bronze_fivetran_salesforce.accounts
26+
WHERE website IS NOT NULL
27+
UNION ALL
28+
SELECT account_id, account_name, website, domain_aliases, NULL AS LOGO_URL, NULL AS INDUSTRY, NULL AS N_EMPLOYEES
29+
FROM analytics.bronze_fivetran_salesforce_b2b.accounts
30+
WHERE website IS NOT NULL
31+
)`
32+
33+
export const buildSourceQuery = (sinceTimestamp?: string): string => {
34+
let select = `
35+
SELECT
36+
ca.*,
37+
co.*,
38+
tu.user_email,
39+
tu.lfid,
40+
tu.learner_name,
41+
tu.user_country,
42+
tu.job_title,
43+
e.project_slug AS PROJECT_SLUG,
44+
e.project_id AS PROJECT_ID,
45+
e.account_id,
46+
org.account_name AS ORGANIZATION_NAME,
47+
org.website AS ORG_WEBSITE,
48+
org.domain_aliases AS ORG_DOMAIN_ALIASES,
49+
org.logo_url AS LOGO_URL,
50+
org.industry AS ORGANIZATION_INDUSTRY,
51+
org.n_employees AS ORGANIZATION_SIZE
52+
FROM analytics.bronze_census_ti.course_actions ca
53+
INNER JOIN analytics.bronze_census_ti.users tu
54+
ON ca.internal_ti_user_id = tu.internal_ti_user_id
55+
INNER JOIN analytics.bronze_census_ti.courses co
56+
ON ca.course_id = co.course_id
57+
INNER JOIN analytics.silver_fact.enrollments e
58+
ON e.course_id = ca.course_id
59+
AND LOWER(e.user_email) = LOWER(tu.user_email)
60+
INNER JOIN cdp_matched_segments cms
61+
ON cms.slug = e.project_slug
62+
AND cms.sourceId = e.project_id
63+
LEFT JOIN org_accounts org
64+
ON e.account_id = org.account_id
65+
WHERE ca.type = 'status_change'
66+
AND ca.source = 'course_started'
67+
AND co.is_test_or_archived = false
68+
AND tu.user_email IS NOT NULL`
69+
70+
if (!IS_PROD_ENV) {
71+
select += ` AND e.project_slug = 'pytorch'`
72+
}
73+
74+
const dedup = `
75+
QUALIFY ROW_NUMBER() OVER (PARTITION BY ca.course_action_id ORDER BY org.website DESC) = 1`
76+
77+
if (!sinceTimestamp) {
78+
return `
79+
WITH ${ORG_ACCOUNTS},
80+
${CDP_MATCHED_SEGMENTS}
81+
${select}
82+
${dedup}`.trim()
83+
}
84+
85+
return `
86+
WITH ${ORG_ACCOUNTS},
87+
${CDP_MATCHED_SEGMENTS},
88+
new_cdp_segments AS (
89+
SELECT DISTINCT
90+
s.SOURCE_ID AS sourceId,
91+
s.slug
92+
FROM ANALYTICS.SILVER_DIM._CROWD_DEV_SEGMENTS_UNION s
93+
WHERE s.CREATED_TS >= '${sinceTimestamp}'
94+
AND s.PARENT_SLUG IS NOT NULL
95+
AND s.GRANDPARENTS_SLUG IS NOT NULL
96+
AND s.SOURCE_ID IS NOT NULL
97+
)
98+
99+
-- New course actions since last export
100+
${select}
101+
AND ca.timestamp >= '${sinceTimestamp}'
102+
${dedup}
103+
104+
UNION
105+
106+
-- All course actions in newly created segments
107+
${select}
108+
AND EXISTS (
109+
SELECT 1 FROM new_cdp_segments ncs
110+
WHERE ncs.slug = cms.slug AND ncs.sourceId = cms.sourceId
111+
)
112+
${dedup}`.trim()
113+
}

0 commit comments

Comments
 (0)