Add custom and pageview goals to exploration suggestions

Author: zoldar

assets/js/dashboard/extra/exploration.js

@@ -48,7 +48,8 @@ function toJourney(steps) {
     name: s.name,
     pathname: s.pathname,
     includes_subpaths: s.includes_subpaths,
-    subpaths_count: s.subpaths_count
+    subpaths_count: s.subpaths_count,
+    is_goal: s.is_goal
   }))
 }
 

extra/lib/plausible/stats/exploration.ex

@@ -8,16 +8,22 @@ defmodule Plausible.Stats.Exploration do
 
     @type t() :: %__MODULE__{}
 
-    @derive {Jason.Encoder, only: [:name, :pathname, :label, :includes_subpaths, :subpaths_count]}
-    defstruct name: nil, pathname: "", label: nil, includes_subpaths: false, subpaths_count: 0
+    @derive {Jason.Encoder,
+             only: [:name, :pathname, :label, :includes_subpaths, :subpaths_count, :is_goal]}
+    defstruct name: nil,
+              pathname: "",
+              label: nil,
+              includes_subpaths: false,
+              subpaths_count: 0,
+              is_goal: false
 
     @spec from(map()) :: t()
     def from(step) do
-      new(step.name, step.pathname, step.includes_subpaths, step.subpaths_count)
+      new(step.name, step.pathname, step.includes_subpaths, step.subpaths_count, step.is_goal)
     end
 
     @spec new(String.t(), String.t(), boolean(), non_neg_integer()) :: t()
-    def new(name, pathname, includes_subpaths \\ false, subpaths_count \\ 0)
+    def new(name, pathname, includes_subpaths \\ false, subpaths_count \\ 0, is_goal \\ false)
         when is_boolean(includes_subpaths) and is_integer(subpaths_count) do
       label =
         if name != "pageview" do
@@ -31,7 +37,8 @@ defmodule Plausible.Stats.Exploration do
         name: name,
         pathname: pathname,
         includes_subpaths: includes_subpaths,
-        subpaths_count: subpaths_count
+        subpaths_count: subpaths_count,
+        is_goal: is_goal
       }
     end
   end
@@ -76,24 +83,29 @@ defmodule Plausible.Stats.Exploration do
   @spec max_steps() :: pos_integer()
   def max_steps, do: @max_steps
 
-  @spec next_steps(Query.t(), journey(), keyword()) ::
+  @spec next_steps(Plausible.Site.t(), Query.t(), journey(), keyword()) ::
           {:ok, [next_step()]} | {:error, :journey_too_long}
-  def next_steps(query, journey, opts \\ [])
+  def next_steps(site, query, journey, opts \\ [])
 
-  def next_steps(_query, journey, _opts) when length(journey) >= @max_steps do
+  def next_steps(_site, _query, journey, _opts) when length(journey) >= @max_steps do
     {:error, :journey_too_long}
   end
 
-  def next_steps(query, journey, opts) do
+  def next_steps(site, query, journey, opts) do
     opts = Keyword.merge(@next_steps_defaults, opts)
     direction = Keyword.fetch!(opts, :direction)
     search_term = Keyword.fetch!(opts, :search_term)
     max_candidates = min(Keyword.fetch!(opts, :max_candidates), @max_candidates)
     include_wilcard? = Keyword.fetch!(opts, :include_wildcard?)
 
+    goals =
+      site
+      |> Plausible.Goals.for_site(include_goals_with_custom_props?: false)
+      |> filter_eligible_goals()
+
     query
     |> Base.base_event_query()
-    |> next_steps_query(journey, search_term, direction, max_candidates, include_wilcard?)
+    |> next_steps_query(journey, search_term, direction, max_candidates, include_wilcard?, goals)
     # We pass the query struct to record query metadata for
     # the CH debug console.
     |> ClickhouseRepo.all(query: query)
@@ -143,9 +155,9 @@ defmodule Plausible.Stats.Exploration do
       to include implicit wildcard pathnames in suggestions or not
       (default: true)
   """
-  @spec interesting_funnel(Query.t(), keyword()) ::
+  @spec interesting_funnel(Plausible.Site.t(), Query.t(), keyword()) ::
           {:ok, %{funnel: [funnel_step()], candidates: [next_step()]}} | {:error, :not_found}
-  def interesting_funnel(query, opts \\ []) do
+  def interesting_funnel(site, query, opts \\ []) do
     max_steps = min(Keyword.get(opts, :max_steps, 6), @max_steps)
     max_candidates = min(Keyword.get(opts, :max_candidates, 10), @max_candidates)
 
@@ -157,14 +169,15 @@ defmodule Plausible.Stats.Exploration do
       )
 
     with {:ok, result} <-
-           build_interesting_journey(query, max_steps, max_candidates, include_wildcard?),
+           build_interesting_journey(site, query, max_steps, max_candidates, include_wildcard?),
          {:ok, funnel} <- journey_funnel(query, result.journey) do
       {:ok, %{funnel: funnel, candidates: result.candidates}}
     end
   end
 
-  defp build_interesting_journey(query, max_steps, max_candidates, include_wildcard?) do
+  defp build_interesting_journey(site, query, max_steps, max_candidates, include_wildcard?) do
     case do_build_journey(
+           site,
            query,
            [],
            [],
@@ -179,6 +192,7 @@ defmodule Plausible.Stats.Exploration do
   end
 
   defp do_build_journey(
+         _site,
          _query,
          journey,
          step_candidates,
@@ -192,6 +206,7 @@ defmodule Plausible.Stats.Exploration do
   end
 
   defp do_build_journey(
+         site,
          query,
          journey,
          step_candidates,
@@ -201,7 +216,7 @@ defmodule Plausible.Stats.Exploration do
          include_wildcard?
        ) do
     {:ok, candidates} =
-      next_steps(query, journey,
+      next_steps(site, query, journey,
         max_candidates: max_candidates,
         include_wildcard?: include_wildcard?
       )
@@ -214,6 +229,7 @@ defmodule Plausible.Stats.Exploration do
         new_seen = MapSet.put(seen, normalize_step_key(step))
 
         do_build_journey(
+          site,
           query,
           journey ++ [step],
           step_candidates ++ [candidates],
@@ -238,7 +254,22 @@ defmodule Plausible.Stats.Exploration do
   defp normalize_pathname("/"), do: "/"
   defp normalize_pathname(pathname), do: String.trim_trailing(pathname, "/")
 
-  defp next_steps_query(query, steps, search_term, direction, max_candidates, include_wildcard?)
+  defp filter_eligible_goals(goals) do
+    Enum.filter(goals, fn g ->
+      is_nil(g.currency) and g.scroll_threshold == -1 and g.custom_props == %{} and
+        is_nil(g.event_name)
+    end)
+  end
+
+  defp next_steps_query(
+         query,
+         steps,
+         search_term,
+         direction,
+         max_candidates,
+         include_wildcard?,
+         goals
+       )
        when is_direction(direction) do
     next_step_idx = length(steps) + 1
     q_steps = steps_query(query, next_step_idx, direction)
@@ -251,7 +282,8 @@ defmodule Plausible.Stats.Exploration do
         where: selected_as(:name) != "",
         select: %{
           name: selected_as(field(s, ^next_name), :name),
-          pathname: selected_as(field(s, ^next_pathname), :pathname)
+          pathname: selected_as(field(s, ^next_pathname), :pathname),
+          _sample_factor: fragment("any(?)", s._sample_factor)
         }
       )
 
@@ -266,12 +298,10 @@ defmodule Plausible.Stats.Exploration do
 
     q_per_user_matches =
       from(m in q_matches,
-        select_merge: %{user_id: m.user_id, _sample_factor: fragment("any(?)", m._sample_factor)},
+        select_merge: %{user_id: m.user_id},
         group_by: [selected_as(:name), selected_as(:pathname), m.user_id]
       )
 
-    q_combined = combined_query(q_per_user_matches, include_wildcard?)
-
     # Fan out each q_combined row into up to two output rows (exact + wildcard)
     # using ARRAY JOIN over a small boolean array.
     #
@@ -280,8 +310,10 @@ defmodule Plausible.Stats.Exploration do
     # subpath, or same visitor count as exact). ARRAY JOIN then emits one or more
     # rows per group. The joined boolean `is_wildcard` selects which values to
     # use for visitors / includes_subpaths / subpaths_count.
-    q_all_matches =
-      from(m in subquery(q_combined),
+    q_wildcard_combined = combined_wildcard_query(q_per_user_matches, include_wildcard?)
+
+    q_wildcard_combined_matches =
+      from(m in subquery(q_wildcard_combined),
         join:
           is_wildcard in fragment(
             """
@@ -300,6 +332,16 @@ defmodule Plausible.Stats.Exploration do
         hints: "ARRAY",
         where: selected_as(:visitors) > 0,
         select: %{
+          label:
+            selected_as(
+              fragment(
+                "if(? != 'pageview', ?, ?)",
+                m.name,
+                m.name,
+                m.pathname
+              ),
+              :label
+            ),
           name: m.name,
           pathname: m.pathname,
           visitors:
@@ -308,27 +350,28 @@ defmodule Plausible.Stats.Exploration do
               :visitors
             ),
           includes_subpaths: fragment("CAST(?, 'Bool')", is_wildcard),
-          subpaths_count: fragment("if(?, ?, 0)", is_wildcard, m.subpaths_count)
+          subpaths_count: fragment("if(?, ?, 0)", is_wildcard, m.subpaths_count),
+          is_goal: fragment("CAST(?, 'Bool')", false)
         }
       )
 
-    from(m in subquery(q_all_matches),
+    q_all_combined_matches =
+      if q_goal_matches = goals_query(q_per_user_matches, goals) do
+        q_wildcard_combined_matches
+        |> union_all(^q_goal_matches)
+      else
+        q_wildcard_combined_matches
+      end
+
+    from(m in subquery(q_all_combined_matches),
       select: %{
         step: %Journey.Step{
-          label:
-            selected_as(
-              fragment(
-                "if(? != 'pageview', ?, ?)",
-                m.name,
-                m.name,
-                m.pathname
-              ),
-              :label
-            ),
+          label: selected_as(m.label, :label),
           name: m.name,
           pathname: m.pathname,
           includes_subpaths: m.includes_subpaths,
-          subpaths_count: m.subpaths_count
+          subpaths_count: m.subpaths_count,
+          is_goal: m.is_goal
         },
         visitors: m.visitors
       },
@@ -342,6 +385,59 @@ defmodule Plausible.Stats.Exploration do
     |> maybe_search(search_term)
   end
 
+  @goal_pathname_condition """
+  if(? LIKE '%*%', 
+    match(?, concat('^', replaceAll(?, '*', '.*'), '$')), 
+    ? = ?
+  )
+  """
+
+  defp goals_query(_, []), do: nil
+
+  defp goals_query(q_matches, goals) do
+    values =
+      Enum.map(goals, fn g ->
+        %{
+          label: g.display_name,
+          name: g.event_name || "pageview",
+          pathname: Regex.escape(g.page_path || "")
+        }
+      end)
+
+    types = %{label: :string, name: :string, pathname: :string}
+
+    query =
+      from(g in values(values, types),
+        inner_join: m in subquery(q_matches),
+        on:
+          g.name == m.name and
+            (g.name != "pageview" or
+               (g.name == "pageview" and
+                  fragment(
+                    @goal_pathname_condition,
+                    g.pathname,
+                    m.pathname,
+                    g.pathname,
+                    m.pathname,
+                    g.pathname
+                  ))),
+        select: %{
+          label: selected_as(g.label, :label),
+          name: selected_as(g.name, :name),
+          pathname: selected_as(g.pathname, :pathname),
+          visitors: scale_sample(fragment("uniq(?)", m.user_id)),
+          includes_subpaths: fragment("CAST(?, 'Bool')", false),
+          subpaths_count: 0,
+          is_goal: fragment("CAST(?, 'Bool')", true)
+        },
+        group_by: [selected_as(:label), selected_as(:name), selected_as(:pathname)]
+      )
+
+    from(m in subquery(query),
+      where: m.visitors > 0
+    )
+  end
+
   # Expand each (name, pathname, user_id) row into all prefix paths via
   # ARRAY JOIN, then aggregate once to get both exact and wildcard visitor
   # counts in a single scan of events_v2.
@@ -361,7 +457,7 @@ defmodule Plausible.Stats.Exploration do
     arraySlice(split_pathname, 1, 1)), [?])
   """
 
-  defp combined_query(q_matches, true = _include_wildcard?) do
+  defp combined_wildcard_query(q_matches, true = _include_wildcard?) do
     from(em in subquery(q_matches),
       join: pname in fragment(@wildcard_array_join, em.name, em.pathname, em.pathname),
       on: true,
@@ -380,7 +476,7 @@ defmodule Plausible.Stats.Exploration do
     )
   end
 
-  defp combined_query(q_matches, false = _include_wildcard?) do
+  defp combined_wildcard_query(q_matches, false = _include_wildcard?) do
     from(em in subquery(q_matches),
       where: em.name != "pageview" or selected_as(:pathname) != "",
       select: %{

lib/plausible_web/controllers/api/stats_controller.ex

@@ -151,7 +151,7 @@ defmodule PlausibleWeb.Api.StatsController do
            include_wildcard? =
              not FunWithFlags.enabled?(@exploration_wildcard_disabled_flag, for: site),
            {:ok, next_steps} <-
-             Exploration.next_steps(query, journey,
+             Exploration.next_steps(site, query, journey,
                search_term: search_term,
                direction: direction,
                include_wildcard?: include_wildcard?
@@ -188,7 +188,7 @@ defmodule PlausibleWeb.Api.StatsController do
       include_wildcard? =
         not FunWithFlags.enabled?(@exploration_wildcard_disabled_flag, for: site)
 
-      case Exploration.interesting_funnel(query,
+      case Exploration.interesting_funnel(site, query,
              max_steps: params["max_steps"],
              max_candidates: params["max_candidates"],
              include_wildcard?: include_wildcard?
@@ -209,7 +209,7 @@ defmodule PlausibleWeb.Api.StatsController do
            include_wildcard? =
              not FunWithFlags.enabled?(@exploration_wildcard_disabled_flag, for: site),
            {:ok, next_steps} <-
-             Exploration.next_steps(query, journey,
+             Exploration.next_steps(site, query, journey,
                search_term: search_term,
                direction: direction,
                include_wildcard?: include_wildcard?
@@ -242,13 +242,15 @@ defmodule PlausibleWeb.Api.StatsController do
            "name" => name,
            "pathname" => pathname,
            "includes_subpaths" => includes_subpaths,
-           "subpaths_count" => subpaths_count
+           "subpaths_count" => subpaths_count,
+           "is_goal" => is_goal
          }) do
       Exploration.Journey.Step.new(
         name,
         pathname,
         includes_subpaths,
-        subpaths_count
+        subpaths_count,
+        is_goal
       )
     end