Skip to content

Commit b462c2c

Browse files
nicosuavenicosuave
authored
Merge pull request #14 from sidequery/hogflare-sidemantic-model
Add semantic model and person snapshot import
2 parents 884ddad + 4f3f144 commit b462c2c

14 files changed

Lines changed: 2545 additions & 21 deletions

‎docs/import-posthog.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
Hogflare includes a host-side importer for backfilling an existing PostHog project into the same Cloudflare Pipeline sink used by the Worker. It reads PostHog's private API with a personal API key, then writes normalized rows to the pipeline:
44

5-
- persons as `$identify` rows
5+
- persons as `$identify` event rows and, when configured, `PersonPipelineRecord` snapshots
66
- groups as `$groupidentify` rows
77
- historical events from HogQL with original `timestamp`, `created_at`, and PostHog event `uuid` when available
88

@@ -42,7 +42,10 @@ export IMPORT_STATE_FILE=".hogflare-import-state.jsonl"
4242
export IMPORT_TARGET_ACCOUNT_ID="<cloudflare_account_id>"
4343
export IMPORT_TARGET_BUCKET="<r2_bucket>"
4444
export IMPORT_TARGET_TABLE="default.hogflare_events_v3"
45+
export IMPORT_PERSONS_TARGET_TABLE="default.hogflare_persons_v2"
4546
export WRANGLER_R2_SQL_AUTH_TOKEN="<r2 sql token>"
47+
export CLOUDFLARE_PERSONS_PIPELINE_ENDPOINT="https://<persons-stream-id>.ingest.cloudflare.com"
48+
export CLOUDFLARE_PERSONS_PIPELINE_AUTH_TOKEN="<persons pipeline token>" # falls back to event pipeline token
4649
export IMPORT_CLOUDFLARE_API_TOKEN="<token with Pipelines read>" # optional auto flush discovery
4750
export IMPORT_PIPELINE_FLUSH_SECS="300" # fallback if Pipelines read is unavailable
4851
```
@@ -75,6 +78,8 @@ cargo run --bin import_posthog -- \
7578
--personal-api-key "$POSTHOG_PERSONAL_API_KEY" \
7679
--pipeline-endpoint "$CLOUDFLARE_PIPELINE_ENDPOINT" \
7780
--pipeline-auth-token "$CLOUDFLARE_PIPELINE_AUTH_TOKEN" \
81+
--persons-pipeline-endpoint "$CLOUDFLARE_PERSONS_PIPELINE_ENDPOINT" \
82+
--persons-pipeline-auth-token "$CLOUDFLARE_PERSONS_PIPELINE_AUTH_TOKEN" \
7883
--hogflare-api-key phc_example \
7984
--from 2025-01-01 \
8085
--to 2025-02-01 \
@@ -91,6 +96,7 @@ cargo run --bin import_posthog -- \
9196
--target-account-id "$CLOUDFLARE_ACCOUNT_ID" \
9297
--target-bucket hogflare \
9398
--target-table default.hogflare_events_v3 \
99+
--persons-target-table default.hogflare_persons_v2 \
94100
--target-auth-token "$WRANGLER_R2_SQL_AUTH_TOKEN" \
95101
--cloudflare-api-token "$CLOUDFLARE_API_TOKEN"
96102
```

‎models/activity_days.yml‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
models:
2+
- name: activity_days
3+
sql: |
4+
with latest_profiles as (
5+
select *
6+
from (
7+
select
8+
*,
9+
row_number() over (partition by person_id order by updated_at desc, version desc, created_at desc) as profile_rank
10+
from {{ persons_table }}
11+
where person_id is not null
12+
)
13+
where profile_rank = 1
14+
),
15+
identity_map as (
16+
select
17+
resolved_person_id,
18+
canonical_distinct_id,
19+
linked_distinct_id
20+
from (
21+
select
22+
person_id as resolved_person_id,
23+
canonical_distinct_id,
24+
distinct_id as linked_distinct_id,
25+
row_number() over (partition by distinct_id order by updated_at desc, version desc, created_at desc) as identity_rank
26+
from latest_profiles,
27+
unnest(distinct_ids) as identity_distinct_ids(distinct_id)
28+
where distinct_id is not null
29+
)
30+
where identity_rank = 1
31+
),
32+
event_base as (
33+
select
34+
uuid,
35+
event as event_type,
36+
distinct_id as actor_id,
37+
person_id,
38+
coalesce(person_id, identity_map.resolved_person_id) as resolved_person_id,
39+
identity_map.canonical_distinct_id,
40+
coalesce(timestamp, created_at) as event_time,
41+
date_trunc('day', coalesce(timestamp, created_at)) as activity_day,
42+
properties,
43+
api_key,
44+
coalesce(json_extract_string(properties, '$.$session_id'), json_extract_string(properties, '$.session_id')) as session_id,
45+
coalesce(json_extract_string(properties, '$.$pathname'), json_extract_string(properties, '$.pathname'), json_extract_string(properties, '$.path')) as pathname,
46+
coalesce(json_extract_string(properties, '$.$current_url'), json_extract_string(properties, '$.current_url'), json_extract_string(properties, '$.url')) as current_url,
47+
coalesce(json_extract_string(properties, '$.$referrer'), json_extract_string(properties, '$.referrer')) as referrer,
48+
coalesce(json_extract_string(properties, '$.$utm_source'), json_extract_string(properties, '$.utm_source'), json_extract_string(properties, '$.$initial_utm_source')) as utm_source,
49+
coalesce(json_extract_string(properties, '$.$utm_medium'), json_extract_string(properties, '$.utm_medium'), json_extract_string(properties, '$.$initial_utm_medium')) as utm_medium,
50+
coalesce(json_extract_string(properties, '$.$utm_campaign'), json_extract_string(properties, '$.utm_campaign'), json_extract_string(properties, '$.$initial_utm_campaign')) as utm_campaign,
51+
coalesce(json_extract_string(properties, '$.$browser'), json_extract_string(properties, '$.browser')) as browser,
52+
coalesce(json_extract_string(properties, '$.$os'), json_extract_string(properties, '$.os')) as os,
53+
coalesce(json_extract_string(properties, '$.$device_type'), json_extract_string(properties, '$.device_type')) as device_type,
54+
coalesce(json_extract_string(properties, '$.$geoip_country_code'), json_extract_string(properties, '$.country')) as geo_country_code
55+
from {{ events_table }}
56+
left join identity_map on distinct_id = identity_map.linked_distinct_id
57+
where coalesce(timestamp, created_at) is not null
58+
),
59+
fallback_events as (
60+
select *
61+
from event_base
62+
where session_id is null
63+
)
64+
select
65+
actor_id || ':' || strftime(activity_day, '%Y-%m-%d') as activity_day_key,
66+
actor_id,
67+
max(resolved_person_id) as person_id,
68+
max(canonical_distinct_id) as canonical_distinct_id,
69+
first(api_key order by event_time asc) as api_key,
70+
activity_day,
71+
min(event_time) as first_activity_at,
72+
max(event_time) as last_activity_at,
73+
date_diff('second', min(event_time), max(event_time)) as duration_seconds,
74+
count(*) as event_count,
75+
sum(case when event_type in ('$pageview', 'page_view', '$screen', 'screen') then 1 else 0 end) as pageview_count,
76+
sum(case when event_type = '$autocapture' then 1 else 0 end) as autocapture_count,
77+
first(pathname order by event_time asc) as first_path,
78+
last(pathname order by event_time asc) as last_path,
79+
first(current_url order by event_time asc) as first_url,
80+
last(current_url order by event_time asc) as last_url,
81+
first(referrer order by event_time asc) as referrer,
82+
first(utm_source order by event_time asc) as utm_source,
83+
first(utm_medium order by event_time asc) as utm_medium,
84+
first(utm_campaign order by event_time asc) as utm_campaign,
85+
first(browser order by event_time asc) as browser,
86+
first(os order by event_time asc) as os,
87+
first(device_type order by event_time asc) as device_type,
88+
first(geo_country_code order by event_time asc) as geo_country_code
89+
from fallback_events
90+
group by actor_id, activity_day
91+
description: Actor/day fallback activity rollup for events that did not carry a PostHog $session_id.
92+
primary_key: activity_day_key
93+
default_time_dimension: activity_day
94+
default_grain: day
95+
96+
relationships:
97+
- name: persons
98+
type: many_to_one
99+
foreign_key: actor_id
100+
primary_key: actor_id
101+
102+
dimensions:
103+
- name: activity_day_key
104+
type: categorical
105+
- name: actor_id
106+
type: categorical
107+
- name: person_id
108+
type: categorical
109+
- name: canonical_distinct_id
110+
type: categorical
111+
- name: api_key
112+
type: categorical
113+
- name: first_path
114+
type: categorical
115+
- name: last_path
116+
type: categorical
117+
- name: first_url
118+
type: categorical
119+
- name: last_url
120+
type: categorical
121+
- name: referrer
122+
type: categorical
123+
- name: utm_source
124+
type: categorical
125+
- name: utm_medium
126+
type: categorical
127+
- name: utm_campaign
128+
type: categorical
129+
- name: browser
130+
type: categorical
131+
- name: os
132+
type: categorical
133+
- name: device_type
134+
type: categorical
135+
- name: geo_country_code
136+
type: categorical
137+
- name: duration_seconds
138+
type: numeric
139+
- name: event_count
140+
type: numeric
141+
- name: pageview_count
142+
type: numeric
143+
- name: activity_day
144+
type: time
145+
granularity: day
146+
- name: first_activity_at
147+
type: time
148+
granularity: day
149+
- name: last_activity_at
150+
type: time
151+
granularity: day
152+
153+
metrics:
154+
- name: activity_day_count
155+
agg: count
156+
description: Actor/day fallback rows for events without SDK session ids.
157+
- name: active_users
158+
agg: count_distinct
159+
sql: actor_id
160+
description: Distinct actors with fallback activity days.
161+
- name: fallback_events
162+
agg: sum
163+
sql: event_count
164+
description: Events included in actor/day fallback rollups.
165+
- name: fallback_pageviews
166+
agg: sum
167+
sql: pageview_count
168+
description: Pageviews included in actor/day fallback rollups.
169+
- name: bounced_activity_days
170+
agg: count
171+
filters:
172+
- pageview_count <= 1
173+
description: Fallback actor/day rows with one or fewer pageviews.
174+
- name: engaged_activity_days
175+
agg: count
176+
filters:
177+
- event_count > 1 or duration_seconds >= 10
178+
description: Fallback actor/day rows with multiple events or at least 10 seconds duration.
179+
- name: total_activity_seconds
180+
agg: sum
181+
sql: duration_seconds
182+
description: Total duration across fallback actor/day rollups.
183+

0 commit comments

Comments
 (0)