diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts index ea22a6824..c43ee7eb0 100644 --- a/backend/apps/cloud/src/analytics/analytics.controller.ts +++ b/backend/apps/cloud/src/analytics/analytics.controller.ts @@ -536,11 +536,15 @@ export class AnalyticsController { } })(), (async () => { - timeToConvert = await this.analyticsService.getFunnelTimeToConvert( - pagesArr, - params, - filtersQuery, - ) + try { + timeToConvert = await this.analyticsService.getFunnelTimeToConvert( + pagesArr, + params, + filtersQuery, + ) + } catch (e) { + this.logger.error(e, 'GET /analytics/funnel - getFunnelTimeToConvert') + } })(), ] @@ -1068,14 +1072,31 @@ export class AnalyticsController { const params = { pid, groupFrom, groupTo, ...filtersParams } - const flow = await this.analyticsService.getJourneys( - params, - filtersQuery, - steps, - journeys, - ) + let nodeDetails = [] + let linkDetails = [] + + const [flow] = await Promise.all([ + this.analyticsService.getJourneys(params, filtersQuery, steps, journeys), + (async () => { + try { + const details = await this.analyticsService.getJourneyNodeDetails( + params, + filtersQuery, + steps, + journeys, + ) + nodeDetails = details.nodes + linkDetails = details.links + } catch (e) { + this.logger.error( + e, + 'GET /analytics/journeys - getJourneyNodeDetails', + ) + } + })(), + ]) - return { ...flow, appliedFilters } + return { ...flow, nodeDetails, linkDetails, appliedFilters } } @Get('journey-sessions') diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index c70e4b190..08cde15b7 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -96,6 +96,8 @@ import { GetFiltersQuery, IJourney, IJourneys, + IJourneyNodeDetails, + IJourneyLinkDetails, IExtractChartData, IGenerateXAxis, IAggregatedMetadata, @@ -149,6 +151,10 @@ const MAX_FILTER_VALUES = 100 // session while keeping a bot's session array bounded. const MAX_JOURNEY_PAGEVIEWS_PER_SESSION = 1000 +// Synthetic link target marking sessions whose journey ended at the source +// node; must match the frontend's Sankey exit node identifier. +const JOURNEY_EXIT_TARGET = '__exit__' + // Event types that can be targeted by the data-deletion tool. const DELETABLE_EVENT_TYPES = [...DATA_DELETION_EVENT_TYPES] // The subset that actually lives in the `events` table. Session replays are @@ -1168,20 +1174,22 @@ export class AnalyticsService { // are excluded. On top of the overall top-{journeys} ranking, the top 3 // paths at every depth are kept so that long journeys remain visible even // when short paths dominate the ranking. - const query = ` - WITH session_paths AS ( + // + // fullLength (the compacted path length before arraySlice) is kept so + // that sessions which ENDED at the last drawn step can be distinguished + // from sessions which CONTINUED past it (truncated by the steps limit). + const sessionPathsCTE = ` + session_paths AS ( SELECT psid, - arraySlice( - arrayCompact( - arrayMap( - x -> x.2, - groupArraySorted({maxPageviews:UInt32})((created, pg)) - ) - ), - 1, - {steps:UInt32} - ) AS path + arrayCompact( + arrayMap( + x -> x.2, + groupArraySorted({maxPageviews:UInt32})((created, pg)) + ) + ) AS fullPath, + arraySlice(fullPath, 1, {steps:UInt32}) AS path, + length(fullPath) AS fullLength FROM events WHERE pid = {pid:FixedString(12)} @@ -1191,23 +1199,32 @@ export class AnalyticsService { ${sessionFiltersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psid - HAVING length(path) >= 2 + HAVING fullLength >= 2 ) + ` + + const query = ` + WITH ${sessionPathsCTE} SELECT path, value, - totalSessions + continuedPast, + totalSessions, + totalPaths FROM ( SELECT path, value, + continuedPast, sum(value) OVER () AS totalSessions, - row_number() OVER (ORDER BY value DESC) AS overallRank, - row_number() OVER (PARTITION BY length(path) ORDER BY value DESC) AS depthRank + count() OVER () AS totalPaths, + row_number() OVER (ORDER BY value DESC, path ASC) AS overallRank, + row_number() OVER (PARTITION BY length(path) ORDER BY value DESC, path ASC) AS depthRank FROM ( SELECT path, - count() AS value + count() AS value, + countIf(fullLength > {steps:UInt32}) AS continuedPast FROM session_paths GROUP BY path ) @@ -1216,6 +1233,225 @@ export class AnalyticsService { ORDER BY value DESC ` + // How many sessions have a (sliced) path of each length. Lets the client + // compute, per step, how many sessions reached that step at all -- the + // basis for the "Other paths" and "Exited" nodes and the coverage line. + const histogramQuery = ` + WITH ${sessionPathsCTE} + SELECT + length(path) AS len, + count() AS sessions, + countIf(fullLength > {steps:UInt32}) AS truncated + FROM session_paths + GROUP BY len + ORDER BY len + ` + + const queryParams = { + ...params, + steps, + journeys, + maxPageviews: MAX_JOURNEY_PAGEVIEWS_PER_SESSION, + } + + const [{ data }, { data: histogramData }] = await Promise.all([ + clickhouse.query({ query, query_params: queryParams }).then((res) => + res.json<{ + path: string[] + value: string + continuedPast: string + totalSessions: string + totalPaths: string + }>(), + ), + clickhouse + .query({ query: histogramQuery, query_params: queryParams }) + .then((res) => + res.json<{ len: string; sessions: string; truncated: string }>(), + ), + ]) + + if (_isEmpty(data)) { + return { + journeys: [], + totalSessions: 0, + totalPaths: 0, + lengthHistogram: [], + } + } + + const journeyList: IJourney[] = _map(data, (row) => ({ + path: row.path, + value: Number(row.value), + continuedPast: Number(row.continuedPast), + })) + + return { + journeys: journeyList, + totalSessions: Number(data[0].totalSessions), + totalPaths: Number(data[0].totalPaths), + lengthHistogram: _map(histogramData, (row) => ({ + len: Number(row.len), + sessions: Number(row.sessions), + truncated: Number(row.truncated), + })), + } + } + + // Top sources & countries for every node AND link drawn in the journeys + // Sankey. Computed over the TRUE population (every session whose path + // passes through the node / makes the transition), matching what the + // journey sessions drawer shows -- not just the sessions on top-N drawn + // paths. Links with target '__exit__' describe sessions whose journey + // ended at the source node. + async getJourneyNodeDetails( + params: Record, + filtersQuery: string, + steps: number, + journeys: number, + ): Promise<{ + nodes: IJourneyNodeDetails[] + links: IJourneyLinkDetails[] + }> { + const sessionFiltersQuery = filtersQuery + ? `AND psid IN ( + SELECT DISTINCT psid + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid != 0 + ${filtersQuery} + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + )` + : '' + + const query = ` + WITH session_paths AS ( + SELECT + psid, + arrayCompact( + arrayMap( + x -> x.2, + groupArraySorted({maxPageviews:UInt32})((created, pg)) + ) + ) AS fullPath, + arraySlice(fullPath, 1, {steps:UInt32}) AS path, + length(fullPath) AS fullLength + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type = 'pageview' + AND psid != 0 + AND pg IS NOT NULL + ${sessionFiltersQuery} + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + GROUP BY psid + HAVING fullLength >= 2 + ), + ranked_paths AS ( + SELECT path + FROM ( + SELECT + path, + count() AS value, + row_number() OVER (ORDER BY value DESC, path ASC) AS overallRank, + row_number() OVER (PARTITION BY length(path) ORDER BY value DESC, path ASC) AS depthRank + FROM session_paths + GROUP BY path + ) + WHERE overallRank <= {journeys:UInt32} OR depthRank <= 3 + ), + drawn_nodes AS ( + SELECT DISTINCT idx - 1 AS step, path[idx] AS page + FROM ranked_paths + ARRAY JOIN arrayEnumerate(path) AS idx + ), + node_sessions AS ( + SELECT sp.psid AS psid, idx - 1 AS step, sp.path[idx] AS page + FROM session_paths sp + ARRAY JOIN arrayEnumerate(sp.path) AS idx + WHERE (idx - 1, sp.path[idx]) IN (SELECT step, page FROM drawn_nodes) + ), + drawn_links AS ( + SELECT DISTINCT + idx - 1 AS step, + path[idx] AS source, + if(idx < length(path), path[idx + 1], '${JOURNEY_EXIT_TARGET}') AS target + FROM ranked_paths + ARRAY JOIN arrayEnumerate(path) AS idx + ), + link_sessions AS ( + SELECT + sp.psid AS psid, + idx - 1 AS step, + sp.path[idx] AS source, + if(idx < length(sp.path), sp.path[idx + 1], '${JOURNEY_EXIT_TARGET}') AS target + FROM session_paths sp + ARRAY JOIN arrayEnumerate(sp.path) AS idx + WHERE (idx < length(sp.path) OR sp.fullLength = length(sp.path)) + AND ( + idx - 1, + sp.path[idx], + if(idx < length(sp.path), sp.path[idx + 1], '${JOURNEY_EXIT_TARGET}') + ) IN (SELECT step, source, target FROM drawn_links) + ), + session_info AS ( + SELECT + psid, + argMin(cc, created) AS cc, + argMin(if(so IS NOT NULL AND so != '', so, if(domain(ref) != '', domain(ref), 'Direct / None')), created) AS source + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' + AND psid != 0 + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + GROUP BY psid + ) + SELECT step, page, target, type, val, cnt FROM ( + SELECT ns.step AS step, ns.page AS page, '' AS target, 'countries' AS type, si.cc AS val, count() AS cnt + FROM node_sessions ns + INNER JOIN session_info si ON ns.psid = si.psid + WHERE si.cc != '' + GROUP BY ns.step, ns.page, si.cc + + UNION ALL + + SELECT ns.step AS step, ns.page AS page, '' AS target, 'sources' AS type, si.source AS val, count() AS cnt + FROM node_sessions ns + INNER JOIN session_info si ON ns.psid = si.psid + GROUP BY ns.step, ns.page, si.source + + UNION ALL + + SELECT ns.step AS step, ns.page AS page, '' AS target, 'total' AS type, '' AS val, count() AS cnt + FROM node_sessions ns + GROUP BY ns.step, ns.page + + UNION ALL + + SELECT ls.step AS step, ls.source AS page, ls.target AS target, 'countries' AS type, si.cc AS val, count() AS cnt + FROM link_sessions ls + INNER JOIN session_info si ON ls.psid = si.psid + WHERE si.cc != '' + GROUP BY ls.step, ls.source, ls.target, si.cc + + UNION ALL + + SELECT ls.step AS step, ls.source AS page, ls.target AS target, 'sources' AS type, si.source AS val, count() AS cnt + FROM link_sessions ls + INNER JOIN session_info si ON ls.psid = si.psid + GROUP BY ls.step, ls.source, ls.target, si.source + + UNION ALL + + SELECT ls.step AS step, ls.source AS page, ls.target AS target, 'total' AS type, '' AS val, count() AS cnt + FROM link_sessions ls + GROUP BY ls.step, ls.source, ls.target + ) + ORDER BY step, page, target, type, cnt DESC, val ASC + LIMIT 5 BY step, page, target, type + ` + const { data } = await clickhouse .query({ query, @@ -1227,21 +1463,63 @@ export class AnalyticsService { }, }) .then((res) => - res.json<{ path: string[]; value: string; totalSessions: string }>(), + res.json<{ + step: string + page: string + target: string + type: 'countries' | 'sources' | 'total' + val: string + cnt: string + }>(), ) - if (_isEmpty(data)) { - return { journeys: [], totalSessions: 0 } - } + const nodeMap = new Map() + const linkMap = new Map() - const journeyList: IJourney[] = _map(data, (row) => ({ - path: row.path, - value: Number(row.value), - })) + for (const row of data) { + const step = Number(row.step) + + let entry: IJourneyNodeDetails | IJourneyLinkDetails + + if (row.target === '') { + const key = `${step}:${row.page}` + let node = nodeMap.get(key) + + if (!node) { + node = { step, page: row.page, total: 0, sources: {}, countries: {} } + nodeMap.set(key, node) + } + + entry = node + } else { + const key = `${step}:${row.page}→${row.target}` + let link = linkMap.get(key) + + if (!link) { + link = { + step, + source: row.page, + target: row.target, + total: 0, + sources: {}, + countries: {}, + } + linkMap.set(key, link) + } + + entry = link + } + + if (row.type === 'total') { + entry.total = Number(row.cnt) + } else { + entry[row.type][row.val] = Number(row.cnt) + } + } return { - journeys: journeyList, - totalSessions: Number(data[0].totalSessions), + nodes: Array.from(nodeMap.values()), + links: Array.from(linkMap.values()), } } diff --git a/backend/apps/cloud/src/analytics/interfaces/index.ts b/backend/apps/cloud/src/analytics/interfaces/index.ts index fd93e2aa5..31d80943e 100644 --- a/backend/apps/cloud/src/analytics/interfaces/index.ts +++ b/backend/apps/cloud/src/analytics/interfaces/index.ts @@ -86,11 +86,37 @@ export interface IGenerateXAxis { export interface IJourney { path: string[] value: number + // of `value`, how many sessions continued past the last drawn step + // (i.e. were truncated by the steps limit rather than ending there) + continuedPast: number } export interface IJourneys { journeys: IJourney[] + // multi-page sessions only (single-page bounces produce no transitions) totalSessions: number + // distinct paths before the top-N ranking was applied + totalPaths: number + // how many sessions have a (sliced) path of each length + lengthHistogram: { len: number; sessions: number; truncated: number }[] +} + +export interface IJourneyNodeDetails { + step: number + page: string + total: number + sources: Record + countries: Record +} + +export interface IJourneyLinkDetails { + step: number + source: string + // next page, or '__exit__' when the journey ended at the source node + target: string + total: number + sources: Record + countries: Record } export interface IExtractChartData { diff --git a/backend/apps/cloud/src/analytics/v2/analytics-v2.service.ts b/backend/apps/cloud/src/analytics/v2/analytics-v2.service.ts index 3d7756d4a..7da1e041e 100644 --- a/backend/apps/cloud/src/analytics/v2/analytics-v2.service.ts +++ b/backend/apps/cloud/src/analytics/v2/analytics-v2.service.ts @@ -1705,11 +1705,18 @@ export class AnalyticsV2Service { } })(), (async () => { - timeToConvert = await this.analyticsService.getFunnelTimeToConvert( - steps, - params, - filters.filtersQuery, - ) + try { + timeToConvert = await this.analyticsService.getFunnelTimeToConvert( + steps, + params, + filters.filtersQuery, + ) + } catch (reason) { + this.logger.error( + reason, + 'AnalyticsV2Service -> getFunnelTimeToConvert', + ) + } })(), ]) diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.ts index 61bc50b40..62ef26360 100644 --- a/backend/apps/community/src/analytics/analytics.controller.ts +++ b/backend/apps/community/src/analytics/analytics.controller.ts @@ -500,11 +500,15 @@ export class AnalyticsController { } })(), (async () => { - timeToConvert = await this.analyticsService.getFunnelTimeToConvert( - pagesArr, - params, - filtersQuery, - ) + try { + timeToConvert = await this.analyticsService.getFunnelTimeToConvert( + pagesArr, + params, + filtersQuery, + ) + } catch (e) { + this.logger.error(e, 'GET /analytics/funnel - getFunnelTimeToConvert') + } })(), ] @@ -932,14 +936,31 @@ export class AnalyticsController { const params = { pid, groupFrom, groupTo, ...filtersParams } - const flow = await this.analyticsService.getJourneys( - params, - filtersQuery, - steps, - journeys, - ) + let nodeDetails = [] + let linkDetails = [] + + const [flow] = await Promise.all([ + this.analyticsService.getJourneys(params, filtersQuery, steps, journeys), + (async () => { + try { + const details = await this.analyticsService.getJourneyNodeDetails( + params, + filtersQuery, + steps, + journeys, + ) + nodeDetails = details.nodes + linkDetails = details.links + } catch (e) { + this.logger.error( + e, + 'GET /analytics/journeys - getJourneyNodeDetails', + ) + } + })(), + ]) - return { ...flow, appliedFilters: parsedFilters } + return { ...flow, nodeDetails, linkDetails, appliedFilters: parsedFilters } } @Get('journey-sessions') diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index e6c43c064..c00dd5dfc 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -82,6 +82,8 @@ import { GetFiltersQuery, IJourney, IJourneys, + IJourneyNodeDetails, + IJourneyLinkDetails, IExtractChartData, IGenerateXAxis, IAggregatedMetadata, @@ -132,6 +134,10 @@ const MAX_FILTER_VALUES = 100 // session while keeping a bot's session array bounded. const MAX_JOURNEY_PAGEVIEWS_PER_SESSION = 1000 +// Synthetic link target marking sessions whose journey ended at the source +// node; must match the frontend's Sankey exit node identifier. +const JOURNEY_EXIT_TARGET = '__exit__' + // Event types that can be targeted by the data-deletion tool. const DELETABLE_EVENT_TYPES = [...DATA_DELETION_EVENT_TYPES] // Sentinel bounds used when no explicit date range is given, so that @@ -971,20 +977,22 @@ export class AnalyticsService { // are excluded. On top of the overall top-{journeys} ranking, the top 3 // paths at every depth are kept so that long journeys remain visible even // when short paths dominate the ranking. - const query = ` - WITH session_paths AS ( + // + // fullLength (the compacted path length before arraySlice) is kept so + // that sessions which ENDED at the last drawn step can be distinguished + // from sessions which CONTINUED past it (truncated by the steps limit). + const sessionPathsCTE = ` + session_paths AS ( SELECT psid, - arraySlice( - arrayCompact( - arrayMap( - x -> x.2, - groupArraySorted({maxPageviews:UInt32})((created, pg)) - ) - ), - 1, - {steps:UInt32} - ) AS path + arrayCompact( + arrayMap( + x -> x.2, + groupArraySorted({maxPageviews:UInt32})((created, pg)) + ) + ) AS fullPath, + arraySlice(fullPath, 1, {steps:UInt32}) AS path, + length(fullPath) AS fullLength FROM events WHERE pid = {pid:FixedString(12)} @@ -994,23 +1002,32 @@ export class AnalyticsService { ${sessionFiltersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psid - HAVING length(path) >= 2 + HAVING fullLength >= 2 ) + ` + + const query = ` + WITH ${sessionPathsCTE} SELECT path, value, - totalSessions + continuedPast, + totalSessions, + totalPaths FROM ( SELECT path, value, + continuedPast, sum(value) OVER () AS totalSessions, - row_number() OVER (ORDER BY value DESC) AS overallRank, - row_number() OVER (PARTITION BY length(path) ORDER BY value DESC) AS depthRank + count() OVER () AS totalPaths, + row_number() OVER (ORDER BY value DESC, path ASC) AS overallRank, + row_number() OVER (PARTITION BY length(path) ORDER BY value DESC, path ASC) AS depthRank FROM ( SELECT path, - count() AS value + count() AS value, + countIf(fullLength > {steps:UInt32}) AS continuedPast FROM session_paths GROUP BY path ) @@ -1019,6 +1036,225 @@ export class AnalyticsService { ORDER BY value DESC ` + // How many sessions have a (sliced) path of each length. Lets the client + // compute, per step, how many sessions reached that step at all -- the + // basis for the "Other paths" and "Exited" nodes and the coverage line. + const histogramQuery = ` + WITH ${sessionPathsCTE} + SELECT + length(path) AS len, + count() AS sessions, + countIf(fullLength > {steps:UInt32}) AS truncated + FROM session_paths + GROUP BY len + ORDER BY len + ` + + const queryParams = { + ...params, + steps, + journeys, + maxPageviews: MAX_JOURNEY_PAGEVIEWS_PER_SESSION, + } + + const [{ data }, { data: histogramData }] = await Promise.all([ + clickhouse.query({ query, query_params: queryParams }).then((res) => + res.json<{ + path: string[] + value: string + continuedPast: string + totalSessions: string + totalPaths: string + }>(), + ), + clickhouse + .query({ query: histogramQuery, query_params: queryParams }) + .then((res) => + res.json<{ len: string; sessions: string; truncated: string }>(), + ), + ]) + + if (_isEmpty(data)) { + return { + journeys: [], + totalSessions: 0, + totalPaths: 0, + lengthHistogram: [], + } + } + + const journeyList: IJourney[] = _map(data, (row) => ({ + path: row.path, + value: Number(row.value), + continuedPast: Number(row.continuedPast), + })) + + return { + journeys: journeyList, + totalSessions: Number(data[0].totalSessions), + totalPaths: Number(data[0].totalPaths), + lengthHistogram: _map(histogramData, (row) => ({ + len: Number(row.len), + sessions: Number(row.sessions), + truncated: Number(row.truncated), + })), + } + } + + // Top sources & countries for every node AND link drawn in the journeys + // Sankey. Computed over the TRUE population (every session whose path + // passes through the node / makes the transition), matching what the + // journey sessions drawer shows -- not just the sessions on top-N drawn + // paths. Links with target '__exit__' describe sessions whose journey + // ended at the source node. + async getJourneyNodeDetails( + params: Record, + filtersQuery: string, + steps: number, + journeys: number, + ): Promise<{ + nodes: IJourneyNodeDetails[] + links: IJourneyLinkDetails[] + }> { + const sessionFiltersQuery = filtersQuery + ? `AND psid IN ( + SELECT DISTINCT psid + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid != 0 + ${filtersQuery} + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + )` + : '' + + const query = ` + WITH session_paths AS ( + SELECT + psid, + arrayCompact( + arrayMap( + x -> x.2, + groupArraySorted({maxPageviews:UInt32})((created, pg)) + ) + ) AS fullPath, + arraySlice(fullPath, 1, {steps:UInt32}) AS path, + length(fullPath) AS fullLength + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type = 'pageview' + AND psid != 0 + AND pg IS NOT NULL + ${sessionFiltersQuery} + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + GROUP BY psid + HAVING fullLength >= 2 + ), + ranked_paths AS ( + SELECT path + FROM ( + SELECT + path, + count() AS value, + row_number() OVER (ORDER BY value DESC, path ASC) AS overallRank, + row_number() OVER (PARTITION BY length(path) ORDER BY value DESC, path ASC) AS depthRank + FROM session_paths + GROUP BY path + ) + WHERE overallRank <= {journeys:UInt32} OR depthRank <= 3 + ), + drawn_nodes AS ( + SELECT DISTINCT idx - 1 AS step, path[idx] AS page + FROM ranked_paths + ARRAY JOIN arrayEnumerate(path) AS idx + ), + node_sessions AS ( + SELECT sp.psid AS psid, idx - 1 AS step, sp.path[idx] AS page + FROM session_paths sp + ARRAY JOIN arrayEnumerate(sp.path) AS idx + WHERE (idx - 1, sp.path[idx]) IN (SELECT step, page FROM drawn_nodes) + ), + drawn_links AS ( + SELECT DISTINCT + idx - 1 AS step, + path[idx] AS source, + if(idx < length(path), path[idx + 1], '${JOURNEY_EXIT_TARGET}') AS target + FROM ranked_paths + ARRAY JOIN arrayEnumerate(path) AS idx + ), + link_sessions AS ( + SELECT + sp.psid AS psid, + idx - 1 AS step, + sp.path[idx] AS source, + if(idx < length(sp.path), sp.path[idx + 1], '${JOURNEY_EXIT_TARGET}') AS target + FROM session_paths sp + ARRAY JOIN arrayEnumerate(sp.path) AS idx + WHERE (idx < length(sp.path) OR sp.fullLength = length(sp.path)) + AND ( + idx - 1, + sp.path[idx], + if(idx < length(sp.path), sp.path[idx + 1], '${JOURNEY_EXIT_TARGET}') + ) IN (SELECT step, source, target FROM drawn_links) + ), + session_info AS ( + SELECT + psid, + argMin(cc, created) AS cc, + argMin(if(so IS NOT NULL AND so != '', so, if(domain(ref) != '', domain(ref), 'Direct / None')), created) AS source + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' + AND psid != 0 + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + GROUP BY psid + ) + SELECT step, page, target, type, val, cnt FROM ( + SELECT ns.step AS step, ns.page AS page, '' AS target, 'countries' AS type, si.cc AS val, count() AS cnt + FROM node_sessions ns + INNER JOIN session_info si ON ns.psid = si.psid + WHERE si.cc != '' + GROUP BY ns.step, ns.page, si.cc + + UNION ALL + + SELECT ns.step AS step, ns.page AS page, '' AS target, 'sources' AS type, si.source AS val, count() AS cnt + FROM node_sessions ns + INNER JOIN session_info si ON ns.psid = si.psid + GROUP BY ns.step, ns.page, si.source + + UNION ALL + + SELECT ns.step AS step, ns.page AS page, '' AS target, 'total' AS type, '' AS val, count() AS cnt + FROM node_sessions ns + GROUP BY ns.step, ns.page + + UNION ALL + + SELECT ls.step AS step, ls.source AS page, ls.target AS target, 'countries' AS type, si.cc AS val, count() AS cnt + FROM link_sessions ls + INNER JOIN session_info si ON ls.psid = si.psid + WHERE si.cc != '' + GROUP BY ls.step, ls.source, ls.target, si.cc + + UNION ALL + + SELECT ls.step AS step, ls.source AS page, ls.target AS target, 'sources' AS type, si.source AS val, count() AS cnt + FROM link_sessions ls + INNER JOIN session_info si ON ls.psid = si.psid + GROUP BY ls.step, ls.source, ls.target, si.source + + UNION ALL + + SELECT ls.step AS step, ls.source AS page, ls.target AS target, 'total' AS type, '' AS val, count() AS cnt + FROM link_sessions ls + GROUP BY ls.step, ls.source, ls.target + ) + ORDER BY step, page, target, type, cnt DESC, val ASC + LIMIT 5 BY step, page, target, type + ` + const { data } = await clickhouse .query({ query, @@ -1030,21 +1266,63 @@ export class AnalyticsService { }, }) .then((res) => - res.json<{ path: string[]; value: string; totalSessions: string }>(), + res.json<{ + step: string + page: string + target: string + type: 'countries' | 'sources' | 'total' + val: string + cnt: string + }>(), ) - if (_isEmpty(data)) { - return { journeys: [], totalSessions: 0 } - } + const nodeMap = new Map() + const linkMap = new Map() - const journeyList: IJourney[] = _map(data, (row) => ({ - path: row.path, - value: Number(row.value), - })) + for (const row of data) { + const step = Number(row.step) + + let entry: IJourneyNodeDetails | IJourneyLinkDetails + + if (row.target === '') { + const key = `${step}:${row.page}` + let node = nodeMap.get(key) + + if (!node) { + node = { step, page: row.page, total: 0, sources: {}, countries: {} } + nodeMap.set(key, node) + } + + entry = node + } else { + const key = `${step}:${row.page}→${row.target}` + let link = linkMap.get(key) + + if (!link) { + link = { + step, + source: row.page, + target: row.target, + total: 0, + sources: {}, + countries: {}, + } + linkMap.set(key, link) + } + + entry = link + } + + if (row.type === 'total') { + entry.total = Number(row.cnt) + } else { + entry[row.type][row.val] = Number(row.cnt) + } + } return { - journeys: journeyList, - totalSessions: Number(data[0].totalSessions), + nodes: Array.from(nodeMap.values()), + links: Array.from(linkMap.values()), } } @@ -2581,19 +2859,19 @@ export class AnalyticsService { funnel_sessions AS ( SELECT psid, - argMin(firstStepAt, conversionAt) AS firstStepAt, - min(conversionAt) AS conversionAt + argMin(firstStepAt, rawConversionAt) AS firstStepAt, + min(rawConversionAt) AS conversionAt FROM ( SELECT psid, firstStepAt, - step${finalStep}At AS conversionAt + step${finalStep}At AS rawConversionAt FROM funnel_step_${finalStep} ) GROUP BY psid ), session_starts AS ( - SELECT fs.psid, s.firstSeen AS sessionStart + SELECT fs.psid AS ssPsid, s.firstSeen AS sessionStart FROM funnel_sessions fs INNER JOIN ( SELECT psid, firstSeen, lastSeen @@ -2604,9 +2882,9 @@ export class AnalyticsService { AND s.lastSeen >= fs.conversionAt ), first_pages AS ( - SELECT fs.psid, min(e.created) AS firstPageAt + SELECT fs.psid AS fpPsid, min(e.created) AS firstPageAt FROM funnel_sessions fs - INNER JOIN session_starts ss ON fs.psid = ss.psid + INNER JOIN session_starts ss ON fs.psid = ss.ssPsid INNER JOIN events e ON e.psid = fs.psid WHERE e.pid = {pid:FixedString(12)} AND e.type = 'pageview' @@ -2626,8 +2904,8 @@ export class AnalyticsService { quantileExactOrNull(0.5)(if(firstStepAt > toDateTime(0) AND firstStepAt <= conversionAt, dateDiff('second', firstStepAt, conversionAt), NULL)) AS medianFirstStep, quantileExactOrNull(0.75)(if(firstStepAt > toDateTime(0) AND firstStepAt <= conversionAt, dateDiff('second', firstStepAt, conversionAt), NULL)) AS p75FirstStep FROM funnel_sessions fs - LEFT JOIN session_starts ss ON fs.psid = ss.psid - LEFT JOIN first_pages fp ON fs.psid = fp.psid + LEFT JOIN session_starts ss ON fs.psid = ss.ssPsid + LEFT JOIN first_pages fp ON fs.psid = fp.fpPsid ` const { data } = await clickhouse diff --git a/backend/apps/community/src/analytics/interfaces/index.ts b/backend/apps/community/src/analytics/interfaces/index.ts index a0279566b..ccd1337a2 100644 --- a/backend/apps/community/src/analytics/interfaces/index.ts +++ b/backend/apps/community/src/analytics/interfaces/index.ts @@ -78,11 +78,37 @@ export interface IGenerateXAxis { export interface IJourney { path: string[] value: number + // of `value`, how many sessions continued past the last drawn step + // (i.e. were truncated by the steps limit rather than ending there) + continuedPast: number } export interface IJourneys { journeys: IJourney[] + // multi-page sessions only (single-page bounces produce no transitions) totalSessions: number + // distinct paths before the top-N ranking was applied + totalPaths: number + // how many sessions have a (sliced) path of each length + lengthHistogram: { len: number; sessions: number; truncated: number }[] +} + +export interface IJourneyNodeDetails { + step: number + page: string + total: number + sources: Record + countries: Record +} + +export interface IJourneyLinkDetails { + step: number + source: string + // next page, or '__exit__' when the journey ended at the source node + target: string + total: number + sources: Record + countries: Record } export interface IExtractChartData { diff --git a/backend/apps/community/src/analytics/v2/analytics-v2.service.ts b/backend/apps/community/src/analytics/v2/analytics-v2.service.ts index 55606aeaa..4e71a3b24 100644 --- a/backend/apps/community/src/analytics/v2/analytics-v2.service.ts +++ b/backend/apps/community/src/analytics/v2/analytics-v2.service.ts @@ -1704,11 +1704,18 @@ export class AnalyticsV2Service { } })(), (async () => { - timeToConvert = await this.analyticsService.getFunnelTimeToConvert( - steps, - params, - filters.filtersQuery, - ) + try { + timeToConvert = await this.analyticsService.getFunnelTimeToConvert( + steps, + params, + filters.filtersQuery, + ) + } catch (reason) { + this.logger.log( + reason, + 'AnalyticsV2Service -> getFunnelTimeToConvert', + ) + } })(), ]) diff --git a/web/app/api/api.server.ts b/web/app/api/api.server.ts index e9a0dd13a..18c1893b1 100644 --- a/web/app/api/api.server.ts +++ b/web/app/api/api.server.ts @@ -1703,11 +1703,38 @@ export async function processSSOTokenServer( interface Journey { path: string[] value: number + // of `value`, how many sessions continued past the last drawn step + continuedPast: number +} + +export interface JourneyNodeDetails { + step: number + page: string + total: number + sources: Record + countries: Record +} + +export interface JourneyLinkDetails { + step: number + source: string + // next page, or '__exit__' when the journey ended at the source node + target: string + total: number + sources: Record + countries: Record } export interface JourneysResponse { journeys: Journey[] + // multi-page sessions only (single-page bounces produce no transitions) totalSessions: number + // distinct paths before the top-N ranking was applied + totalPaths: number + // how many sessions have a (sliced) path of each length + lengthHistogram: { len: number; sessions: number; truncated: number }[] + nodeDetails: JourneyNodeDetails[] + linkDetails: JourneyLinkDetails[] } export async function getJourneysServer( diff --git a/web/app/pages/Project/View/ViewProject.helpers.tsx b/web/app/pages/Project/View/ViewProject.helpers.tsx index 8303bea75..03d86dd97 100644 --- a/web/app/pages/Project/View/ViewProject.helpers.tsx +++ b/web/app/pages/Project/View/ViewProject.helpers.tsx @@ -1827,8 +1827,14 @@ const getSettingsFunnels = ( const prevStep = index > 0 ? funnel[index - 1] : null const prevStepTitle = escapeHtml(values[index - 1] ?? '') - const topSourcesEntries = Object.entries(step.topSources || {}) - const topCountriesEntries = Object.entries(step.topCountries || {}) + // the v2 stats API ships per-step breakdowns; topSources/topCountries + // are the legacy v1 fields kept as a fallback + const topSourcesEntries = Object.entries( + step.breakdowns?.sources || step.topSources || {}, + ) + const topCountriesEntries = Object.entries( + step.breakdowns?.countries || step.topCountries || {}, + ) const primary = 'text-gray-900 dark:text-gray-50' const secondary = 'text-gray-700 dark:text-gray-200' diff --git a/web/app/pages/Project/tabs/Journeys/JourneysView.tsx b/web/app/pages/Project/tabs/Journeys/JourneysView.tsx index ad2f42a0e..e2a507753 100644 --- a/web/app/pages/Project/tabs/Journeys/JourneysView.tsx +++ b/web/app/pages/Project/tabs/Journeys/JourneysView.tsx @@ -1,12 +1,19 @@ import { ResponsiveSankey } from '@nivo/sankey' -import { PathIcon } from '@phosphor-icons/react' +import { FlowArrowIcon, PathIcon, TableIcon } from '@phosphor-icons/react' +import cx from 'clsx' import _isEmpty from 'lodash/isEmpty' import { useEffect, useMemo, useState } from 'react' -import { useTranslation } from 'react-i18next' +import { Trans, useTranslation } from 'react-i18next' +import { useSearchParams } from 'react-router' import { toast } from 'sonner' -import type { JourneysResponse } from '~/api/api.server' +import type { + JourneyLinkDetails, + JourneyNodeDetails, + JourneysResponse, +} from '~/api/api.server' import { useJourneysProxy } from '~/hooks/useAnalyticsProxy' +import { DOCS_URL } from '~/lib/constants' import DashboardHeader from '~/pages/Project/View/components/DashboardHeader' import Filters from '~/pages/Project/View/components/Filters' import ProjectViewHeaderActions from '~/pages/Project/View/components/ProjectViewHeaderActions' @@ -20,10 +27,13 @@ import { useTheme } from '~/providers/ThemeProvider' import Loader from '~/ui/Loader' import LoadingBar from '~/ui/LoadingBar' import { Text } from '~/ui/Text' +import countries from '~/utils/isoCountries' import { SessionsDrawer } from '../Traffic/SessionsDrawer' import { getFormatDate } from '../../View/ViewProject.helpers' +const JOURNEYS_DOCS_URL = `${DOCS_URL}/analytics-dashboard/journeys` + const MIN_STEPS = 2 const MAX_STEPS = 10 const MIN_JOURNEYS = 5 @@ -31,6 +41,11 @@ const MAX_JOURNEYS = 100 const DEFAULT_STEPS = 3 const DEFAULT_JOURNEYS = 20 +const TOOLTIP_BREAKDOWN_LIMIT = 3 + +// Colours are assigned to PAGES (by traffic volume), not to steps -- the same +// page keeps the same colour in every column so it can be traced across the +// diagram. Position already encodes the step. const NODE_COLOURS = [ '#2563eb', '#0d9488', @@ -44,22 +59,195 @@ const NODE_COLOURS = [ '#b45309', ] +const NEUTRAL_PAGE_COLOUR = '#64748b' + +const OTHER_PAGE = '__other__' +const EXIT_PAGE = '__exit__' + +const otherId = (step: number) => `${step}:${OTHER_PAGE}` +const exitId = (step: number) => `${step}:${EXIT_PAGE}` + +type NodeKind = 'page' | 'other' | 'exit' + interface SankeyNodeMeta { page: string step: number + kind: NodeKind sessions: number + // sessions whose journey ended at this node + exited: number + // sessions that continued past the last drawn column at this node + continuedPast: number } interface JourneysViewProps { tnMapping: Record } +// Middle truncation: for real routes the distinguishing token is usually at +// the END of the path (/docs/selfhosting/how-to-install vs ...-update), so +// keep both the prefix and the tail. const truncatePath = (path: string, maxLength = 24) => { if (path.length <= maxLength) { return path } - return `${path.slice(0, maxLength - 1)}…` + const head = Math.ceil((maxLength - 1) * 0.45) + const tail = maxLength - 1 - head + + return `${path.slice(0, head)}…${path.slice(-tail)}` +} + +const clampSteps = (value: number) => + Math.min(MAX_STEPS, Math.max(MIN_STEPS, Math.floor(value))) + +const clampJourneys = (value: number) => + Math.min(MAX_JOURNEYS, Math.max(MIN_JOURNEYS, Math.floor(value))) + +const SLIDER_CLASSNAME = + 'h-2 w-36 cursor-pointer appearance-none rounded-full accent-blue-600 sm:w-44 [&::-moz-range-thumb]:size-4 [&::-moz-range-thumb]:cursor-grab [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:border-2 [&::-moz-range-thumb]:border-blue-600 [&::-moz-range-thumb]:bg-white [&::-moz-range-thumb]:shadow-sm active:[&::-moz-range-thumb]:cursor-grabbing [&::-webkit-slider-thumb]:size-4 [&::-webkit-slider-thumb]:cursor-grab [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:border-2 [&::-webkit-slider-thumb]:border-blue-600 [&::-webkit-slider-thumb]:bg-white [&::-webkit-slider-thumb]:shadow-sm active:[&::-webkit-slider-thumb]:cursor-grabbing' + +// Filled-track gradient, same treatment as the pricing page slider +// (MarketingPricing.tsx). Track colour is theme-dependent because the inline +// background overrides any Tailwind bg-* class. +const sliderTrackStyle = ( + value: number, + min: number, + max: number, + theme: string, +) => { + const percent = max > min ? ((value - min) / (max - min)) * 100 : 0 + const track = theme === 'dark' ? 'rgb(51 65 85)' : 'rgb(209 213 219)' + + return { + background: `linear-gradient(to right, rgb(37 99 235) 0%, rgb(37 99 235) ${percent}%, ${track} ${percent}%, ${track} 100%)`, + } +} + +interface BreakdownDetails { + total: number + sources: Record + countries: Record +} + +interface TooltipBreakdownsProps { + details: BreakdownDetails | undefined +} + +// Same presentation as the goals / funnels tooltip breakdowns: top sources +// and top countries side by side, percentages relative to every session +// passing through the node (or making the transition, for links). +const TooltipBreakdowns = ({ details }: TooltipBreakdownsProps) => { + const { + t, + i18n: { language }, + } = useTranslation('common') + + if (!details) { + return null + } + + const getTopEntries = (items?: Record) => + Object.entries(items || {}) + .filter(([, count]) => Number.isFinite(count) && count > 0) + .sort(([, a], [, b]) => b - a) + .slice(0, TOOLTIP_BREAKDOWN_LIMIT) + + const topSources = getTopEntries(details.sources) + const topCountries = getTopEntries(details.countries) + + if (topSources.length === 0 && topCountries.length === 0) { + return null + } + + const denominator = Math.max(details.total, 1) + + return ( +
+ {topSources.length > 0 ? ( +
+

+ {t('project.topSources')} +

+ {topSources.map(([domain, count]) => { + const perc = Math.round((count / denominator) * 100) + const isDirect = domain === 'Direct / None' + + return ( +
+
+ {isDirect ? ( + <> + + + + ) : ( + + )} + + {domain} + +
+ + {perc}% + +
+ ) + })} +
+ ) : null} + {topCountries.length > 0 ? ( +
+

+ {t('project.topCountries')} +

+ {topCountries.map(([cc, count]) => { + const perc = Math.round((count / denominator) * 100) + + return ( +
+
+ + + {countries.getName(cc, language) || cc} + +
+ + {perc}% + +
+ ) + })} +
+ ) : null} +
+ ) } const JourneysView = ({ tnMapping }: JourneysViewProps) => { @@ -69,12 +257,24 @@ const JourneysView = ({ tnMapping }: JourneysViewProps) => { const { id } = useCurrentProject() const { theme } = useTheme() const { t } = useTranslation('common') + const [searchParams, setSearchParams] = useSearchParams() - const [steps, setSteps] = useState(DEFAULT_STEPS) - const [journeysCount, setJourneysCount] = useState(DEFAULT_JOURNEYS) + const [steps, setSteps] = useState(() => { + const value = Number(searchParams.get('steps')) + return Number.isFinite(value) && value ? clampSteps(value) : DEFAULT_STEPS + }) + const [journeysCount, setJourneysCount] = useState(() => { + const value = Number(searchParams.get('journeys')) + return Number.isFinite(value) && value + ? clampJourneys(value) + : DEFAULT_JOURNEYS + }) + const [view, setView] = useState<'chart' | 'table'>(() => + searchParams.get('view') === 'table' ? 'table' : 'chart', + ) const [debouncedParams, setDebouncedParams] = useState({ - steps: DEFAULT_STEPS, - journeys: DEFAULT_JOURNEYS, + steps, + journeys: journeysCount, }) const [isLoading, setIsLoading] = useState(null) const [journeysData, setJourneysData] = useState( @@ -83,7 +283,7 @@ const JourneysView = ({ tnMapping }: JourneysViewProps) => { const [sessionsDrawer, setSessionsDrawer] = useState<{ step: number page: string - sessions: number + totalSessions?: number } | null>(null) const { fetchJourneys } = useJourneysProxy() @@ -96,6 +296,36 @@ const JourneysView = ({ tnMapping }: JourneysViewProps) => { return [getFormatDate(dateRange[0]), getFormatDate(dateRange[1])] }, [dateRange]) + const syncSearchParams = (updates: Record) => { + const next = new URLSearchParams(searchParams.toString()) + + for (const [key, value] of Object.entries(updates)) { + next.set(key, value) + } + + setSearchParams(next, { replace: true, preventScrollReset: true }) + } + + // the URL can change without remounting the component (back/forward + // navigation, shared links), so the controls have to follow it + useEffect(() => { + const stepsValue = Number(searchParams.get('steps')) + setSteps( + Number.isFinite(stepsValue) && stepsValue + ? clampSteps(stepsValue) + : DEFAULT_STEPS, + ) + + const journeysValue = Number(searchParams.get('journeys')) + setJourneysCount( + Number.isFinite(journeysValue) && journeysValue + ? clampJourneys(journeysValue) + : DEFAULT_JOURNEYS, + ) + + setView(searchParams.get('view') === 'table' ? 'table' : 'chart') + }, [searchParams]) + useEffect(() => { const timer = setTimeout(() => { setDebouncedParams({ steps, journeys: journeysCount }) @@ -156,20 +386,64 @@ const JourneysView = ({ tnMapping }: JourneysViewProps) => { journeysRefreshTrigger, ]) - const { sankeyData, nodeMeta, maxColumnNodes } = useMemo(() => { + const nodeDetailsMap = useMemo(() => { + const map = new Map() + + for (const details of journeysData?.nodeDetails || []) { + map.set(`${details.step}:${details.page}`, details) + } + + return map + }, [journeysData]) + + const linkDetailsMap = useMemo(() => { + const map = new Map() + + for (const details of journeysData?.linkDetails || []) { + map.set(`${details.step}:${details.source}→${details.target}`, details) + } + + return map + }, [journeysData]) + + const { + sankeyData, + nodeMeta, + pageColours, + maxColumnNodes, + columns, + coverage, + } = useMemo(() => { const meta = new Map() + const colours = new Map() + + const empty = { + sankeyData: null, + nodeMeta: meta, + pageColours: colours, + maxColumnNodes: 0, + columns: 0, + coverage: null, + } if (_isEmpty(journeysData?.journeys)) { - return { sankeyData: null, nodeMeta: meta, maxColumnNodes: 0 } + return empty } + const { + journeys, + totalSessions, + totalPaths, + lengthHistogram = [], + } = journeysData! + const nodeSessions = new Map() const links = new Map< string, { source: string; target: string; value: number } >() - for (const { path, value } of journeysData!.journeys) { + for (const { path, value, continuedPast = 0 } of journeys) { path.forEach((page, index) => { const nodeId = `${index}:${page}` const node = nodeSessions.get(nodeId) @@ -177,7 +451,14 @@ const JourneysView = ({ tnMapping }: JourneysViewProps) => { if (node) { node.sessions += value } else { - nodeSessions.set(nodeId, { page, step: index, sessions: value }) + nodeSessions.set(nodeId, { + page, + step: index, + kind: 'page', + sessions: value, + exited: 0, + continuedPast: 0, + }) } if (index < path.length - 1) { @@ -192,14 +473,154 @@ const JourneysView = ({ tnMapping }: JourneysViewProps) => { } } }) + + // Terminal accounting: sessions on this path either ended at its last + // node or (only possible at the steps limit) continued past it. + const lastNode = nodeSessions.get( + `${path.length - 1}:${path[path.length - 1]}`, + )! + lastNode.exited += value - continuedPast + lastNode.continuedPast += continuedPast } if (links.size === 0) { - return { sankeyData: null, nodeMeta: meta, maxColumnNodes: 0 } + return empty + } + + // Sessions whose (sliced) path is at least `step + 1` pages long, i.e. + // every session that reached this column -- drawn or not. + const reachedStep = (step: number) => + lengthHistogram.reduce( + (sum, bucket) => (bucket.len > step ? sum + bucket.sessions : sum), + 0, + ) + + const lastColumn = + Math.max(...lengthHistogram.map((bucket) => bucket.len), 2) - 1 + + const drawnAt = new Map() + for (const node of nodeSessions.values()) { + drawnAt.set(node.step, (drawnAt.get(node.step) || 0) + node.sessions) + } + + // "Other paths": sessions that reached this column on a path that is + // not drawn. Queried population minus drawn -- NOT a residual of the + // ribbons, so top-N truncation cannot masquerade as drop-off. + const otherAt = (step: number) => + Math.max(reachedStep(step) - (drawnAt.get(step) || 0), 0) + + // Exits from drawn pages: node@step -> Exited@(step + 1). Exits at the + // last column stay in the node itself (there is no next column) and are + // explained in its tooltip instead. + const exitInflow = new Map() + + for (const node of nodeSessions.values()) { + if (node.exited > 0 && node.step < lastColumn) { + const linkId = `${node.step}:${node.page}→${exitId(node.step + 1)}` + links.set(linkId, { + source: `${node.step}:${node.page}`, + target: exitId(node.step + 1), + value: node.exited, + }) + exitInflow.set( + node.step + 1, + (exitInflow.get(node.step + 1) || 0) + node.exited, + ) + } + } + + // "Other paths" chain. A path is drawn or not drawn wholly, so undrawn + // sessions reaching step + 1 also reached step: the chain is monotone + // and mass-conserving (other[i] = other[i + 1] + exits between them). + const lastColumnBucket = lengthHistogram.find( + (bucket) => bucket.len === lastColumn + 1, + ) + const drawnContinuedAtLast = Array.from(nodeSessions.values()).reduce( + (sum, node) => + node.step === lastColumn ? sum + node.continuedPast : sum, + 0, + ) + + for (let step = 0; step <= lastColumn; step++) { + const sessions = otherAt(step) + + if (sessions <= 0) { + continue + } + + const isLast = step === lastColumn + const continuedPast = isLast + ? Math.max((lastColumnBucket?.truncated || 0) - drawnContinuedAtLast, 0) + : 0 + const nextOther = isLast ? 0 : otherAt(step + 1) + const exited = isLast ? sessions - continuedPast : sessions - nextOther + + nodeSessions.set(otherId(step), { + page: OTHER_PAGE, + step, + kind: 'other', + sessions, + exited: Math.max(exited, 0), + continuedPast, + }) + + if (!isLast && nextOther > 0) { + links.set(`${otherId(step)}→${otherId(step + 1)}`, { + source: otherId(step), + target: otherId(step + 1), + value: nextOther, + }) + } + + if (!isLast && exited > 0) { + links.set(`${otherId(step)}→${exitId(step + 1)}`, { + source: otherId(step), + target: exitId(step + 1), + value: exited, + }) + exitInflow.set(step + 1, (exitInflow.get(step + 1) || 0) + exited) + } + } + + for (const [step, sessions] of exitInflow) { + nodeSessions.set(exitId(step), { + page: EXIT_PAGE, + step, + kind: 'exit', + sessions, + exited: sessions, + continuedPast: 0, + }) + } + + // Stable colour per page (ranked by total traffic across all steps), + // so the same page can be traced across columns. + const pageTotals = new Map() + + for (const node of nodeSessions.values()) { + if (node.kind !== 'page') { + continue + } + + pageTotals.set( + node.page, + (pageTotals.get(node.page) || 0) + node.sessions, + ) } + Array.from(pageTotals.entries()) + .sort((a, b) => b[1] - a[1]) + .forEach(([page], index) => { + colours.set( + page, + index < NODE_COLOURS.length + ? NODE_COLOURS[index] + : NEUTRAL_PAGE_COLOUR, + ) + }) + // Nivo requires every node to participate in at least one link, - // so nodes are derived from links (journeys that end early simply taper off) + // so nodes are derived from links const linkedNodeIds = new Set() for (const link of links.values()) { @@ -217,10 +638,27 @@ const JourneysView = ({ tnMapping }: JourneysViewProps) => { columnCounts.set(node.step, (columnCounts.get(node.step) || 0) + 1) } + const coveredSessions = journeys.reduce( + (sum, journey) => sum + journey.value, + 0, + ) + return { sankeyData: { nodes, links: Array.from(links.values()) }, nodeMeta: meta, + pageColours: colours, maxColumnNodes: Math.max(...columnCounts.values()), + columns: lastColumn + 1, + coverage: { + drawnPaths: journeys.length, + totalPaths, + coveredSessions, + totalSessions, + percent: + totalSessions > 0 + ? Math.round((coveredSessions / totalSessions) * 100) + : 0, + }, } }, [journeysData]) @@ -233,9 +671,28 @@ const JourneysView = ({ tnMapping }: JourneysViewProps) => { const nodeColour = (nodeId: string) => { const node = nodeMeta.get(nodeId) - return NODE_COLOURS[(node?.step ?? 0) % NODE_COLOURS.length] + + if (node?.kind === 'exit') { + return theme === 'dark' ? '#4b5563' : '#9ca3af' + } + + if (node?.kind === 'other') { + return theme === 'dark' ? '#64748b' : '#94a3b8' + } + + return (node && pageColours.get(node.page)) || NEUTRAL_PAGE_COLOUR } + const formatPercent = (value: number, of: number) => + of > 0 ? Math.round((value / of) * 10000) / 100 : 0 + + // Same container and row grammar as the funnel / goals billboard.js + // tooltips: label on the left, mono value on the right, bordered sections. + const tooltipContainerClassName = + 'w-max max-w-sm rounded-lg bg-gray-50 px-3 py-2.5 text-xs shadow-lg ring-1 ring-black/10 md:max-w-md md:text-sm dark:bg-slate-900 dark:ring-white/10' + const tooltipRowClassName = 'mt-1 flex items-center justify-between gap-6' + const tooltipLabelClassName = 'text-gray-700 dark:text-gray-200' + const renderNodeTooltip = (nodeId: string) => { const node = nodeMeta.get(nodeId) @@ -243,26 +700,103 @@ const JourneysView = ({ tnMapping }: JourneysViewProps) => { return null } - const percentage = - totalSessions > 0 - ? Math.round((node.sessions / totalSessions) * 10000) / 100 - : 0 + if (node.kind === 'exit') { + return ( +
+
+ + {t('project.journeys.exited')} + + + -{node.sessions.toLocaleString()} + +
+
+ + {t('project.journeys.afterStepX', { x: node.step })} + +
+
+ ) + } + + if (node.kind === 'other') { + return ( +
+
+
+ + {t('project.journeys.otherPaths')} + + + · {t('project.journeys.stepX', { x: node.step + 1 })} + +
+ + {node.sessions.toLocaleString()} + +
+ {node.continuedPast > 0 ? ( +
+ + {t('project.journeys.continuedFurther')} + + + {node.continuedPast.toLocaleString()} + +
+ ) : null} +
+ ) + } + + const details = nodeDetailsMap.get(nodeId) return ( -
-

- {node.page} -

-

- {t('project.journeys.stepX', { x: node.step + 1 })} ·{' '} - {t('project.journeys.xSessions', { - x: node.sessions.toLocaleString(), - })} -

-

- {t('project.journeys.percentOfSessions', { x: percentage })} -

-

+

+
+
+ + {node.page} + + + · {t('project.journeys.stepX', { x: node.step + 1 })} + +
+ + {node.sessions.toLocaleString()} + +
+
+ + {t('project.journeys.ofMultiPageSessions')} + + + {formatPercent(node.sessions, totalSessions)}% + +
+ {node.exited > 0 ? ( +
+ + {t('project.journeys.tableEndedHere')} + + + -{node.exited.toLocaleString()} + +
+ ) : null} + {node.continuedPast > 0 ? ( +
+ + {t('project.journeys.continuedFurther')} + + + {node.continuedPast.toLocaleString()} + +
+ ) : null} + +

{t('project.journeys.clickToInspect')}

@@ -281,27 +815,64 @@ const JourneysView = ({ tnMapping }: JourneysViewProps) => { return null } - const percentage = - source.sessions > 0 - ? Math.round((value / source.sessions) * 10000) / 100 - : 0 + const percentage = formatPercent(value, source.sessions) + const isExit = target.kind === 'exit' + const sourceLabel = + source.kind === 'other' ? t('project.journeys.otherPaths') : source.page + const targetLabel = isExit + ? t('project.journeys.exited') + : target.kind === 'other' + ? t('project.journeys.otherPaths') + : target.page + + // breakdowns exist for page -> page transitions and page -> exit; + // "Other paths" links aggregate many transitions and have none + const details = + source.kind === 'page' && (isExit || target.kind === 'page') + ? linkDetailsMap.get( + `${source.step}:${source.page}→${isExit ? EXIT_PAGE : target.page}`, + ) + : undefined return ( -
-

- {source.page} → {target.page} -

-

- {t('project.journeys.xSessions', { x: value.toLocaleString() })} -

-

- {t('project.journeys.percentContinued', { x: percentage })} -

+
+
+
+ + {sourceLabel} + + → + + {targetLabel} + +
+ + {value.toLocaleString()} + +
+
+ + {isExit + ? t('project.journeys.exitedHere') + : t('project.journeys.continuedThisWay')} + + + {percentage}% + +
+
) } - const sliders = ( + const controls = (
{ weight='medium' className='whitespace-nowrap tabular-nums' > - {t('project.journeys.xSteps', { x: steps })} + {t('project.journeys.depthLabel', { x: steps })} { min={MIN_STEPS} max={MAX_STEPS} value={steps} - onChange={(e) => setSteps(Number(e.target.value))} - className='w-36 sm:w-44' + onChange={(e) => { + const value = clampSteps(Number(e.target.value)) + setSteps(value) + syncSearchParams({ steps: String(value) }) + }} + className={SLIDER_CLASSNAME} + style={sliderTrackStyle(steps, MIN_STEPS, MAX_STEPS, theme)} />
@@ -329,7 +905,7 @@ const JourneysView = ({ tnMapping }: JourneysViewProps) => { weight='medium' className='whitespace-nowrap tabular-nums' > - {t('project.journeys.xJourneys', { x: journeysCount })} + {t('project.journeys.topPathsLabel', { x: journeysCount })} { max={MAX_JOURNEYS} step={5} value={journeysCount} - onChange={(e) => setJourneysCount(Number(e.target.value))} - className='w-36 sm:w-44' + onChange={(e) => { + const value = clampJourneys(Number(e.target.value)) + setJourneysCount(value) + syncSearchParams({ journeys: String(value) }) + }} + className={SLIDER_CLASSNAME} + style={sliderTrackStyle( + journeysCount, + MIN_JOURNEYS, + MAX_JOURNEYS, + theme, + )} />
+
+ + +
+ {coverage ? ( + + {t('project.journeys.coverage', { + paths: coverage.drawnPaths.toLocaleString(), + totalPaths: coverage.totalPaths.toLocaleString(), + covered: coverage.coveredSessions.toLocaleString(), + total: coverage.totalSessions.toLocaleString(), + percent: coverage.percent, + })} + + ) : null}
) + const openDrawerForNode = (node: SankeyNodeMeta) => { + const details = nodeDetailsMap.get(`${node.step}:${node.page}`) + + setSessionsDrawer({ + step: node.step + 1, + page: node.page, + totalSessions: details?.total, + }) + } + + const journeysTable = + journeysData && !_isEmpty(journeysData.journeys) ? ( +
+ + + + + + + + + + + {journeysData.journeys.map(({ path, value, continuedPast = 0 }) => { + const key = path.join('→') + const ended = value - continuedPast + const lastStep = path.length + const lastPage = path[path.length - 1] + + const openRow = () => + openDrawerForNode({ + page: lastPage, + step: lastStep - 1, + kind: 'page', + sessions: value, + exited: 0, + continuedPast: 0, + }) + + return ( + + + + + + + ) + })} + +
+ {t('project.journeys.tablePath')} + + {t('project.journeys.tableSessions')} + + % + + {t('project.journeys.tableEndedHere')} +
+ + + {value.toLocaleString()} + + {formatPercent(value, totalSessions)}% + + {ended.toLocaleString()} +
+
+ ) : null + return ( { ) : ( <> - {sliders} + {controls} {sankeyData ? ( - // nivo hardcodes z-index 10 on its tooltip wrapper, while the sticky - // sidebar and dashboard header sit at z-20 — lift just the tooltip above them -
- nodeColour(node.id)} - nodeOpacity={1} - nodeHoverOthersOpacity={0.35} - nodeThickness={14} - nodeSpacing={22} - nodeBorderWidth={0} - nodeBorderRadius={3} - linkOpacity={theme === 'dark' ? 0.35 : 0.25} - linkHoverOpacity={0.6} - linkHoverOthersOpacity={0.1} - linkContract={2} - enableLinkGradient - label={(node: any) => { - const meta = nodeMeta.get(node.id) - - if (!meta) { - return node.id - } - - // hide labels of tiny nodes to reduce clutter, they are still hoverable - if ( - totalSessions > 0 && - meta.sessions / totalSessions < 0.015 - ) { - return '' - } - - return truncatePath(meta.page) - }} - labelPosition='inside' - labelOrientation='horizontal' - labelPadding={10} - labelTextColor={theme === 'dark' ? '#e2e8f0' : '#334155'} - theme={{ - labels: { - text: { - fontSize: 13, - fontWeight: 500, - fontFamily: 'inherit', - }, - }, - }} - nodeTooltip={({ node }: any) => renderNodeTooltip(node.id)} - linkTooltip={({ link }: any) => - renderLinkTooltip(link.source.id, link.target.id, link.value) - } - onClick={(data: any) => { - // links (source/target present) are not clickable, only nodes are - if (data?.source || !data?.id) { - return - } - - const node = nodeMeta.get(data.id) - - if (!node) { - return - } - - setSessionsDrawer({ - step: node.step + 1, - page: node.page, - sessions: node.sessions, - }) - }} - /> -
+ view === 'table' ? ( + journeysTable + ) : ( +
+ {columns > 1 ? ( + + ) : null} + {/* nivo hardcodes z-index 10 on its tooltip wrapper, while the sticky + sidebar and dashboard header sit at z-20 — lift just the tooltip above them */} +
+ { + // page nodes by traffic desc; Other paths, then Exited + // pinned to the bottom of every column + const rank = (node: any) => { + const kind = nodeMeta.get(node.id)?.kind + return kind === 'exit' ? 2 : kind === 'other' ? 1 : 0 + } + + return rank(a) - rank(b) || b.value - a.value + }} + colors={(node: any) => nodeColour(node.id)} + nodeOpacity={1} + nodeHoverOthersOpacity={0.35} + nodeThickness={14} + nodeSpacing={22} + nodeBorderWidth={0} + nodeBorderRadius={3} + linkOpacity={theme === 'dark' ? 0.35 : 0.25} + linkHoverOpacity={0.6} + linkHoverOthersOpacity={0.1} + linkContract={2} + linkBlendMode='normal' + // no springs: the tooltip must track the cursor instantly, + // like the billboard.js charts elsewhere in the dashboard + animate={false} + ariaLabel={t('dashboard.journeys')} + ariaDescribedBy='journeys-coverage' + label={(node: any) => { + const meta = nodeMeta.get(node.id) + + if (!meta) { + return node.id + } + + if (meta.kind === 'other') { + return t('project.journeys.otherPaths') + } + + if (meta.kind === 'exit') { + return t('project.journeys.exited') + } + + // hide labels of tiny nodes to reduce clutter, they are still hoverable + if ( + totalSessions > 0 && + meta.sessions / totalSessions < 0.015 + ) { + return '' + } + + return truncatePath(meta.page) + }} + labelPosition='inside' + labelOrientation='horizontal' + labelPadding={10} + labelTextColor={theme === 'dark' ? '#e2e8f0' : '#334155'} + theme={{ + labels: { + text: { + fontSize: 13, + fontWeight: 500, + fontFamily: 'inherit', + }, + }, + }} + nodeTooltip={({ node }: any) => renderNodeTooltip(node.id)} + linkTooltip={({ link }: any) => + renderLinkTooltip( + link.source.id, + link.target.id, + link.value, + ) + } + onClick={(data: any) => { + // links (source/target present) are not clickable, only nodes are + if (data?.source || !data?.id) { + return + } + + const node = nodeMeta.get(data.id) + + // synthetic nodes have no session list to inspect + if (!node || node.kind !== 'page') { + return + } + + openDrawerForNode(node) + }} + /> +
+
+ ) ) : (
@@ -461,6 +1267,28 @@ const JourneysView = ({ tnMapping }: JourneysViewProps) => { > {t('project.journeys.noData')} + + + ), + }} + /> +
)} @@ -485,7 +1313,7 @@ const JourneysView = ({ tnMapping }: JourneysViewProps) => { filters={filters} journeyStep={sessionsDrawer?.step} journeyPage={sessionsDrawer?.page} - totalCount={sessionsDrawer?.sessions} + totalCount={sessionsDrawer?.totalSessions} /> ) diff --git a/web/app/root.tsx b/web/app/root.tsx index 3cfb67985..7c73961c1 100644 --- a/web/app/root.tsx +++ b/web/app/root.tsx @@ -552,7 +552,7 @@ export default function App() { href={isSelfhosted ? '/favicon-ce.ico' : '/favicon.ico'} /> {isDevelopment ? ( -