diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.ex index 7b6e3368f5..8c4115c0aa 100644 --- a/extra/lib/plausible/stats/goal/revenue.ex +++ b/extra/lib/plausible/stats/goal/revenue.ex @@ -4,6 +4,8 @@ defmodule Plausible.Stats.Goal.Revenue do """ import Ecto.Query + alias Plausible.Stats.Query + @revenue_metrics [:average_revenue, :total_revenue] def revenue_metrics() do @@ -37,10 +39,10 @@ defmodule Plausible.Stats.Goal.Revenue do """ def get_revenue_tracking_currency(site, query, metrics) do goal_filters = - case query.filters do - %{"event:goal" => {:is, {_, goal_name}}} -> [goal_name] - %{"event:goal" => {:member, list}} -> Enum.map(list, fn {_, goal_name} -> goal_name end) - _any -> [] + case Query.get_filter(query, "event:goal") do + [:is, "event:goal", {_, goal_name}] -> [goal_name] + [:member, "event:goal", list] -> Enum.map(list, fn {_, goal_name} -> goal_name end) + _ -> [] end requested_revenue_metrics? = Enum.any?(metrics, &(&1 in @revenue_metrics)) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 3fd63c87f7..763981aea9 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -74,16 +74,16 @@ defmodule Plausible.Google.API do end end - def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do + def fetch_stats(site, query, limit) do with {:ok, site} <- ensure_search_console_property(site), {:ok, access_token} <- maybe_refresh_token(site.google_auth), {:ok, search_console_filters} <- - SearchConsole.Filters.transform(site.google_auth.property, filters), + SearchConsole.Filters.transform(site.google_auth.property, query.filters), {:ok, stats} <- HTTP.list_stats( access_token, site.google_auth.property, - date_range, + query.date_range, limit, search_console_filters ) do diff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.ex index 4e0e00d875..484ed84390 100644 --- a/lib/plausible/google/search_console/filters.ex +++ b/lib/plausible/google/search_console/filters.ex @@ -3,12 +3,11 @@ defmodule Plausible.Google.SearchConsole.Filters do import Plausible.Stats.Base, only: [page_regex: 1] def transform(property, plausible_filters) do - plausible_filters = Map.drop(plausible_filters, ["visit:source"]) - search_console_filters = Enum.reduce_while(plausible_filters, [], fn plausible_filter, search_console_filters -> case transform_filter(property, plausible_filter) do :unsupported -> {:halt, :unsupported_filters} + :ignore -> {:cont, search_console_filters} search_console_filter -> {:cont, [search_console_filter | search_console_filters]} end end) @@ -20,27 +19,27 @@ defmodule Plausible.Google.SearchConsole.Filters do end end - defp transform_filter(property, {"event:page", filter}) do - transform_filter(property, {"visit:entry_page", filter}) + defp transform_filter(property, [op, "event:page" | rest]) do + transform_filter(property, [op, "visit:entry_page" | rest]) end - defp transform_filter(property, {"visit:entry_page", {:is, page}}) when is_binary(page) do + defp transform_filter(property, [:is, "visit:entry_page", page]) when is_binary(page) do %{dimension: "page", expression: property_url(property, page)} end - defp transform_filter(property, {"visit:entry_page", {:member, pages}}) when is_list(pages) do + defp transform_filter(property, [:member, "visit:entry_page", pages]) when is_list(pages) do expression = Enum.map_join(pages, "|", fn page -> property_url(property, Regex.escape(page)) end) %{dimension: "page", operator: "includingRegex", expression: expression} end - defp transform_filter(property, {"visit:entry_page", {:matches, page}}) when is_binary(page) do + defp transform_filter(property, [:matches, "visit:entry_page", page]) when is_binary(page) do page = page_regex(property_url(property, page)) %{dimension: "page", operator: "includingRegex", expression: page} end - defp transform_filter(property, {"visit:entry_page", {:matches_member, pages}}) + defp transform_filter(property, [:matches_member, "visit:entry_page", pages]) when is_list(pages) do expression = Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end) @@ -48,25 +47,27 @@ defmodule Plausible.Google.SearchConsole.Filters do %{dimension: "page", operator: "includingRegex", expression: expression} end - defp transform_filter(_property, {"visit:screen", {:is, device}}) when is_binary(device) do + defp transform_filter(_property, [:is, "visit:screen", device]) when is_binary(device) do %{dimension: "device", expression: search_console_device(device)} end - defp transform_filter(_property, {"visit:screen", {:member, devices}}) when is_list(devices) do + defp transform_filter(_property, [:member, "visit:screen", devices]) when is_list(devices) do expression = devices |> Enum.join("|") %{dimension: "device", operator: "includingRegex", expression: expression} end - defp transform_filter(_property, {"visit:country", {:is, country}}) when is_binary(country) do + defp transform_filter(_property, [:is, "visit:country", country]) when is_binary(country) do %{dimension: "country", expression: search_console_country(country)} end - defp transform_filter(_property, {"visit:country", {:member, countries}}) + defp transform_filter(_property, [:member, "visit:country", countries]) when is_list(countries) do expression = Enum.map_join(countries, "|", &search_console_country/1) %{dimension: "country", operator: "includingRegex", expression: expression} end + defp transform_filter(_, [_, "visit:source" | _rest]), do: :ignore + defp transform_filter(_, _filter), do: :unsupported defp property_url("sc-domain:" <> domain, page), do: "https://" <> domain <> page diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 991e80431e..73d33f0b24 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -70,10 +70,7 @@ defmodule Plausible.Stats.Aggregate do defp neighbor_aggregate_time_on_page(site, query) do q = from( - e in base_event_query(site, %Query{ - query - | filters: Map.delete(query.filters, "event:page") - }), + e in base_event_query(site, Query.remove_filters(query, ["event:page"])), select: { fragment("? as p", e.pathname), fragment("? as t", e.timestamp), @@ -86,32 +83,32 @@ defmodule Plausible.Stats.Aggregate do where_param_idx = length(base_query_raw_params) {where_clause, where_arg} = - case query.filters["event:page"] do - {:is, page} -> + case Query.get_filter(query, "event:page") do + [:is, _, page] -> {"p = {$#{where_param_idx}:String}", page} - {:is_not, page} -> + [:is_not, _, page] -> {"p != {$#{where_param_idx}:String}", page} - {:member, page} -> + [:member, _, page] -> {"p IN {$#{where_param_idx}:Array(String)}", page} - {:not_member, page} -> + [:not_member, _, page] -> {"p NOT IN {$#{where_param_idx}:Array(String)}", page} - {:matches, expr} -> + [:matches, _, expr] -> regex = page_regex(expr) {"match(p, {$#{where_param_idx}:String})", regex} - {:matches_member, exprs} -> + [:matches_member, _, exprs] -> page_regexes = Enum.map(exprs, &page_regex/1) {"multiMatchAny(p, {$#{where_param_idx}:Array(String)})", page_regexes} - {:not_matches_member, exprs} -> + [:not_matches_member, _, exprs] -> page_regexes = Enum.map(exprs, &page_regex/1) {"not(multiMatchAny(p, {$#{where_param_idx}:Array(String)}))", page_regexes} - {:does_not_match, expr} -> + [:does_not_match, _, expr] -> regex = page_regex(expr) {"not(match(p, {$#{where_param_idx}:String}))", regex} end @@ -148,29 +145,29 @@ defmodule Plausible.Stats.Aggregate do defp window_aggregate_time_on_page(site, query) do windowed_pages_q = - from e in base_event_query(site, %Query{ - query - | filters: Map.delete(query.filters, "event:page") - }), - select: %{ - next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon), - next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon), - timestamp: e.timestamp, - pathname: e.pathname, - session_id: e.session_id - }, - windows: [ - event_horizon: [ - partition_by: e.session_id, - order_by: e.timestamp, - frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING") - ] - ] + from e in base_event_query(site, Query.remove_filters(query, ["event:page"])), + select: %{ + next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon), + next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon), + timestamp: e.timestamp, + pathname: e.pathname, + session_id: e.session_id + }, + windows: [ + event_horizon: [ + partition_by: e.session_id, + order_by: e.timestamp, + frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING") + ] + ] + + event_page_filter = Query.get_filter(query, "event:page") timed_page_transitions_q = from e in Ecto.Query.subquery(windowed_pages_q), group_by: [e.pathname, e.next_pathname, e.session_id], - where: ^Plausible.Stats.Base.dynamic_filter_condition(query, "event:page", :pathname), + where: + ^Plausible.Stats.Filters.WhereBuilder.build_condition(:pathname, event_page_filter), where: e.next_timestamp != 0, select: %{ pathname: e.pathname, diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index fe74fb426b..07d14a7871 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -7,9 +7,6 @@ defmodule Plausible.Stats.Base do alias Plausible.Timezones import Ecto.Query - @no_ref "Direct / None" - @not_set "(not set)" - @uniq_users_expression "toUInt64(round(uniq(?) * any(_sample_factor)))" def base_event_query(site, query) do @@ -30,180 +27,30 @@ defmodule Plausible.Stats.Base do on: e.session_id == sq.session_id ) else - if query.experimental_reduced_joins? do - events_q |> filter_by_visit_props(Filters.event_table_visit_props(), query) - else - events_q - end + events_q end end defp query_events(site, query) do - {first_datetime, last_datetime} = utc_boundaries(query, site) - - q = - from( - e in "events_v2", - where: e.site_id == ^site.id, - where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime - ) + q = from(e in "events_v2", where: ^Filters.WhereBuilder.build(:events, site, query)) on_ee do q = Plausible.Stats.Sampling.add_query_hint(q, query) end - q = - q - |> where([e], ^dynamic_filter_condition(query, "event:page", :pathname)) - |> where([e], ^dynamic_filter_condition(query, "event:hostname", :hostname)) - - q = - case query.filters["event:name"] do - {:is, name} -> - from(e in q, where: e.name == ^name) - - {:member, list} -> - from(e in q, where: e.name in ^list) - - nil -> - q - end - - q = - case query.filters["event:goal"] do - {:is, {:page, path}} -> - from(e in q, where: e.pathname == ^path and e.name == "pageview") - - {:matches, {:page, expr}} -> - regex = page_regex(expr) - - from(e in q, - where: fragment("match(?, ?)", e.pathname, ^regex) and e.name == "pageview" - ) - - {:is, {:event, event}} -> - from(e in q, where: e.name == ^event) - - {:member, clauses} -> - {events, pages} = split_goals(clauses) - - from(e in q, - where: (e.pathname in ^pages and e.name == "pageview") or e.name in ^events - ) - - {:matches_member, clauses} -> - {events, pages} = split_goals(clauses, &page_regex/1) - - event_clause = - if Enum.any?(events) do - dynamic([x], fragment("multiMatchAny(?, ?)", x.name, ^events)) - else - dynamic([x], false) - end - - page_clause = - if Enum.any?(pages) do - dynamic( - [x], - fragment("multiMatchAny(?, ?)", x.pathname, ^pages) and x.name == "pageview" - ) - else - dynamic([x], false) - end - - where_clause = dynamic([], ^event_clause or ^page_clause) - - from(e in q, where: ^where_clause) - - nil -> - q - end - - q = - Enum.reduce( - Query.get_all_filters_by_prefix(query, "event:props"), - q, - &filter_by_custom_prop/2 - ) - q end def query_sessions(site, query) do - {first_datetime, last_datetime} = - utc_boundaries(query, site) - - q = from(s in "sessions_v2", where: s.site_id == ^site.id) - - sessions_q = - if query.experimental_session_count? do - from s in q, where: s.timestamp >= ^first_datetime and s.start < ^last_datetime - else - from s in q, where: s.start >= ^first_datetime and s.start < ^last_datetime - end + q = from(s in "sessions_v2", where: ^Filters.WhereBuilder.build(:sessions, site, query)) on_ee do - sessions_q = Plausible.Stats.Sampling.add_query_hint(sessions_q, query) + q = Plausible.Stats.Sampling.add_query_hint(q, query) end - filter_by_entry_props(sessions_q, query) - |> filter_by_visit_props(Filters.visit_props(), query) + q end - defp filter_by_visit_props(q, visit_props, query) do - Enum.reduce(visit_props, q, fn prop_name, sessions_q -> - filter_key = "visit:" <> prop_name - db_field = String.to_existing_atom(prop_name) - - from(s in sessions_q, - where: ^dynamic_filter_condition(query, filter_key, db_field) - ) - end) - end - - def filter_by_entry_props(sessions_q, query) do - case Query.get_filter_by_prefix(query, "visit:entry_props:") do - nil -> - sessions_q - - {"visit:entry_props:" <> prop_name, filter_value} -> - apply_entry_prop_filter(sessions_q, prop_name, filter_value) - end - end - - def apply_entry_prop_filter(sessions_q, prop_name, {:is, "(none)"}) do - from( - s in sessions_q, - where: not has_key(s, :entry_meta, ^prop_name) - ) - end - - def apply_entry_prop_filter(sessions_q, prop_name, {:is, value}) do - from( - s in sessions_q, - where: - has_key(s, :entry_meta, ^prop_name) and get_by_key(s, :entry_meta, ^prop_name) == ^value - ) - end - - def apply_entry_prop_filter(sessions_q, prop_name, {:is_not, "(none)"}) do - from( - s in sessions_q, - where: has_key(s, :entry_meta, ^prop_name) - ) - end - - def apply_entry_prop_filter(sessions_q, prop_name, {:is_not, value}) do - from( - s in sessions_q, - where: - not has_key(s, :entry_meta, ^prop_name) or - get_by_key(s, :entry_meta, ^prop_name) != ^value - ) - end - - def apply_entry_prop_filter(sessions_q, _, _), do: sessions_q - def select_event_metrics(metrics) do metrics |> Enum.map(&select_event_metric/1) @@ -272,7 +119,9 @@ defmodule Plausible.Stats.Base do end defp select_session_metric(:bounce_rate, query) do - condition = dynamic_filter_condition(query, "event:page", :entry_page) + # :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page. + event_page_filter = Query.get_filter(query, "event:page") + condition = Filters.WhereBuilder.build_condition(:entry_page, event_page_filter) %{ bounce_rate: @@ -358,45 +207,6 @@ defmodule Plausible.Stats.Base do defp select_session_metric(:percentage, _query), do: %{} - def dynamic_filter_condition(query, filter_key, db_field) do - case query && query.filters && query.filters[filter_key] do - {:is, value} -> - value = db_field_val(db_field, value) - dynamic([x], field(x, ^db_field) == ^value) - - {:is_not, value} -> - value = db_field_val(db_field, value) - dynamic([x], field(x, ^db_field) != ^value) - - {:matches_member, glob_exprs} -> - page_regexes = Enum.map(glob_exprs, &page_regex/1) - dynamic([x], fragment("multiMatchAny(?, ?)", field(x, ^db_field), ^page_regexes)) - - {:not_matches_member, glob_exprs} -> - page_regexes = Enum.map(glob_exprs, &page_regex/1) - dynamic([x], fragment("not(multiMatchAny(?, ?))", field(x, ^db_field), ^page_regexes)) - - {:matches, glob_expr} -> - regex = page_regex(glob_expr) - dynamic([x], fragment("match(?, ?)", field(x, ^db_field), ^regex)) - - {:does_not_match, glob_expr} -> - regex = page_regex(glob_expr) - dynamic([x], fragment("not(match(?, ?))", field(x, ^db_field), ^regex)) - - {:member, list} -> - list = Enum.map(list, &db_field_val(db_field, &1)) - dynamic([x], field(x, ^db_field) in ^list) - - {:not_member, list} -> - list = Enum.map(list, &db_field_val(db_field, &1)) - dynamic([x], field(x, ^db_field) not in ^list) - - nil -> - true - end - end - def filter_converted_sessions(db_query, site, query) do if Query.has_event_filters?(query) do converted_sessions = @@ -416,16 +226,6 @@ defmodule Plausible.Stats.Base do end end - defp db_field_val(:source, @no_ref), do: "" - defp db_field_val(:referrer, @no_ref), do: "" - defp db_field_val(:utm_medium, @no_ref), do: "" - defp db_field_val(:utm_source, @no_ref), do: "" - defp db_field_val(:utm_campaign, @no_ref), do: "" - defp db_field_val(:utm_content, @no_ref), do: "" - defp db_field_val(:utm_term, @no_ref), do: "" - defp db_field_val(_, @not_set), do: "" - defp db_field_val(_, val), do: val - defp beginning_of_time(candidate, native_stats_start_at) do if Timex.after?(native_stats_start_at, candidate) do native_stats_start_at @@ -486,92 +286,6 @@ defmodule Plausible.Stats.Base do "^#{escaped}$" end - defp split_goals(clauses, map_fn \\ &Function.identity/1) do - groups = - Enum.group_by(clauses, fn {goal_type, _v} -> goal_type end, fn {_k, val} -> map_fn.(val) end) - - { - Map.get(groups, :event, []), - Map.get(groups, :page, []) - } - end - - defp filter_by_custom_prop({"event:props:" <> prop_name, {:is, "(none)"}}, q) do - from( - e in q, - where: not has_key(e, :meta, ^prop_name) - ) - end - - defp filter_by_custom_prop({"event:props:" <> prop_name, {:is, value}}, q) do - from( - e in q, - where: has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) == ^value - ) - end - - defp filter_by_custom_prop({"event:props:" <> prop_name, {:is_not, "(none)"}}, q) do - from( - e in q, - where: has_key(e, :meta, ^prop_name) - ) - end - - defp filter_by_custom_prop({"event:props:" <> prop_name, {:is_not, value}}, q) do - from( - e in q, - where: not has_key(e, :meta, ^prop_name) or get_by_key(e, :meta, ^prop_name) != ^value - ) - end - - defp filter_by_custom_prop({"event:props:" <> prop_name, {:matches, value}}, q) do - regex = page_regex(value) - - from( - e in q, - where: - has_key(e, :meta, ^prop_name) and - fragment("match(?, ?)", get_by_key(e, :meta, ^prop_name), ^regex) - ) - end - - defp filter_by_custom_prop({"event:props:" <> prop_name, {:member, values}}, q) do - none_value_included = Enum.member?(values, "(none)") - - from( - e in q, - where: - (has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) in ^values) or - (^none_value_included and not has_key(e, :meta, ^prop_name)) - ) - end - - defp filter_by_custom_prop({"event:props:" <> prop_name, {:not_member, values}}, q) do - none_value_included = Enum.member?(values, "(none)") - - from( - e in q, - where: - (has_key(e, :meta, ^prop_name) and - get_by_key(e, :meta, ^prop_name) not in ^values) or - (^none_value_included and - has_key(e, :meta, ^prop_name) and - get_by_key(e, :meta, ^prop_name) not in ^values) or - (not (^none_value_included) and not has_key(e, :meta, ^prop_name)) - ) - end - - defp filter_by_custom_prop({"event:props:" <> prop_name, {:matches_member, clauses}}, q) do - regexes = Enum.map(clauses, &page_regex/1) - - from( - e in q, - where: - has_key(e, :meta, ^prop_name) and - fragment("arrayExists(k -> match(?, k), ?)", get_by_key(e, :meta, ^prop_name), ^regexes) - ) - end - defp total_visitors(site, query) do base_event_query(site, query) |> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id)) @@ -615,7 +329,7 @@ defmodule Plausible.Stats.Base do # filters. def maybe_add_conversion_rate(q, site, query, metrics) do if :conversion_rate in metrics do - total_query = query |> Query.remove_event_filters([:goal, :props]) + total_query = query |> Query.remove_filters(["event:goal", "event:props"]) # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL subquery(q) diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 566da8a071..305d80838b 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -39,7 +39,7 @@ defmodule Plausible.Stats.Breakdown do event_query = %Query{ query - | filters: Map.put(query.filters, "event:name", {:member, events}), + | filters: query.filters ++ [[:member, "event:name", events]], property: "event:name" } @@ -180,7 +180,7 @@ defmodule Plausible.Stats.Breakdown do pages -> query - |> Query.put_filter("visit:entry_page", {:member, Enum.map(pages, & &1[:page])}) + |> Query.put_filter([:member, "visit:entry_page", Enum.map(pages, & &1[:page])]) |> struct!(property: "visit:entry_page") end @@ -256,12 +256,12 @@ defmodule Plausible.Stats.Breakdown do end defp update_hostname(query, visit_prop) do - case query.filters["event:hostname"] do + case Query.get_filter(query, "event:hostname") do nil -> query - some -> - Plausible.Stats.Query.put_filter(query, visit_prop, some) + [op, "event:hostname", value] -> + Plausible.Stats.Query.put_filter(query, [op, visit_prop, value]) end end @@ -353,7 +353,7 @@ defmodule Plausible.Stats.Breakdown do import Ecto.Query windowed_pages_q = - from e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])), + from e in base_event_query(site, Query.remove_filters(query, ["event:page", "event:props"])), select: %{ next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon), next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon), @@ -719,7 +719,8 @@ defmodule Plausible.Stats.Breakdown do metrics ) do if :conversion_rate in metrics do - breakdown_total_visitors_query = query |> Query.remove_event_filters([:goal, :props]) + breakdown_total_visitors_query = + query |> Query.remove_filters(["event:goal", "event:props"]) breakdown_total_visitors_q = breakdown_fn.(site, breakdown_total_visitors_query, [:visitors]) diff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.ex index 8fe69b9d44..5c6df48eb4 100644 --- a/lib/plausible/stats/custom_props.ex +++ b/lib/plausible/stats/custom_props.ex @@ -9,7 +9,7 @@ defmodule Plausible.Stats.CustomProps do def fetch_prop_names(site, query) do case Query.get_filter_by_prefix(query, "event:props:") do - {"event:props:" <> key, _} -> + [_op, "event:props:" <> key | _rest] -> [key] _ -> diff --git a/lib/plausible/stats/email_report.ex b/lib/plausible/stats/email_report.ex index e446e45326..63809e3af8 100644 --- a/lib/plausible/stats/email_report.ex +++ b/lib/plausible/stats/email_report.ex @@ -48,7 +48,7 @@ defmodule Plausible.Stats.EmailReport do defp put_top_5_sources(stats, site, query) do query = query - |> Query.put_filter("visit:source", {:is_not, "Direct / None"}) + |> Query.put_filter([:is_not, "visit:source", "Direct / None"]) |> struct!(property: "visit:source") sources = Stats.breakdown(site, query, [:visitors], {5, 1}) diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex index ee46392dae..6e38044f99 100644 --- a/lib/plausible/stats/filter_suggestions.ex +++ b/lib/plausible/stats/filter_suggestions.ex @@ -152,10 +152,10 @@ defmodule Plausible.Stats.FilterSuggestions do def filter_suggestions(site, query, "prop_value", filter_search) do filter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%" - {"event:props:" <> key, _filter} = Query.get_filter_by_prefix(query, "event:props") + [_op, "event:props:" <> key | _rest] = Query.get_filter_by_prefix(query, "event:props") none_q = - from(e in base_event_query(site, Query.remove_event_filters(query, [:props])), + from(e in base_event_query(site, Query.remove_filters(query, ["event:props"])), select: "(none)", where: not has_key(e, :meta, ^key), limit: 1 diff --git a/lib/plausible/stats/filters/dashboard_filter_parser.ex b/lib/plausible/stats/filters/dashboard_filter_parser.ex index e1f80811c1..e57d1d49cb 100644 --- a/lib/plausible/stats/filters/dashboard_filter_parser.ex +++ b/lib/plausible/stats/filters/dashboard_filter_parser.ex @@ -9,52 +9,76 @@ defmodule Plausible.Stats.Filters.DashboardFilterParser do the internal React dashboard API """ def parse_and_prefix(filters_map) do - Enum.reduce(filters_map, %{}, fn {name, val}, new_filters -> + Enum.flat_map(filters_map, fn {name, val} -> cond do name in Filters.visit_props() -> - Map.put(new_filters, "visit:" <> name, filter_value(name, val)) + [filter_value("visit:" <> name, val)] name in Filters.event_props() -> - Map.put(new_filters, "event:" <> name, filter_value(name, val)) + [filter_value("event:" <> name, val)] name == "props" -> - put_parsed_props(new_filters, name, val) + parse_props(val) true -> - new_filters + [] end end) end - @spec filter_value(String.t(), String.t()) :: {atom(), String.t() | [String.t()]} def filter_value(key, val) do {is_negated, val} = parse_negated_prefix(val) {is_contains, val} = parse_contains_prefix(val) is_list = list_expression?(val) is_wildcard = String.contains?(key, ["page", "goal", "hostname"]) && wildcard_expression?(val) val = if is_list, do: parse_member_list(val), else: remove_escape_chars(val) - val = if key == "goal", do: wrap_goal_value(val), else: val + val = if key == "event:goal", do: wrap_goal_value(val), else: val cond do - is_negated && is_wildcard && is_list -> {:not_matches_member, val} - is_negated && is_contains && is_list -> {:not_matches_member, Enum.map(val, &"**#{&1}**")} - is_wildcard && is_list -> {:matches_member, val} - is_negated && is_wildcard -> {:does_not_match, val} - is_negated && is_list -> {:not_member, val} - is_negated && is_contains -> {:does_not_match, "**" <> val <> "**"} - is_contains && is_list -> {:matches_member, Enum.map(val, &"**#{&1}**")} - is_wildcard && is_list -> {:matches_member, val} - is_negated -> {:is_not, val} - is_list -> {:member, val} - is_contains -> {:matches, "**" <> val <> "**"} - is_wildcard -> {:matches, val} - true -> {:is, val} + is_negated && is_wildcard && is_list -> + [:not_matches_member, key, val] + + is_negated && is_contains && is_list -> + [:not_matches_member, key, Enum.map(val, &"**#{&1}**")] + + is_wildcard && is_list -> + [:matches_member, key, val] + + is_negated && is_wildcard -> + [:does_not_match, key, val] + + is_negated && is_list -> + [:not_member, key, val] + + is_negated && is_contains -> + [:does_not_match, key, "**" <> val <> "**"] + + is_contains && is_list -> + [:matches_member, key, Enum.map(val, &"**#{&1}**")] + + is_wildcard && is_list -> + [:matches_member, key, val] + + is_negated -> + [:is_not, key, val] + + is_list -> + [:member, key, val] + + is_contains -> + [:matches, key, "**" <> val <> "**"] + + is_wildcard -> + [:matches, key, val] + + true -> + [:is, key, val] end end - defp put_parsed_props(new_filters, name, val) do - Enum.reduce(val, new_filters, fn {prop_key, prop_val}, new_filters -> - Map.put(new_filters, "event:props:" <> prop_key, filter_value(name, prop_val)) + defp parse_props(val) do + Enum.map(val, fn {prop_key, prop_val} -> + filter_value("event:props:" <> prop_key, prop_val) end) end diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex index 7323696864..3775e73435 100644 --- a/lib/plausible/stats/filters/filters.ex +++ b/lib/plausible/stats/filters/filters.ex @@ -56,24 +56,24 @@ defmodule Plausible.Stats.Filters do ### Examples: iex> Filters.parse("{\\"page\\":\\"/blog/**\\"}") - %{"event:page" => {:matches, "/blog/**"}} + [[:matches, "event:page", "/blog/**"]] iex> Filters.parse("visit:browser!=Chrome") - %{"visit:browser" => {:is_not, "Chrome"}} + [[:is_not, "visit:browser", "Chrome"]] iex> Filters.parse(nil) - %{} + [] """ def parse(filters) when is_binary(filters) do case Jason.decode(filters) do {:ok, filters} when is_map(filters) -> DashboardFilterParser.parse_and_prefix(filters) - {:ok, _} -> %{} + {:ok, _} -> [] {:error, err} -> StatsAPIFilterParser.parse_filter_expression(err.data) end end def parse(filters) when is_map(filters), do: DashboardFilterParser.parse_and_prefix(filters) - def parse(_), do: %{} + def parse(_), do: [] def without_prefix(property) do property diff --git a/lib/plausible/stats/filters/stats_api_filter_parser.ex b/lib/plausible/stats/filters/stats_api_filter_parser.ex index b946e5950a..fc58168993 100644 --- a/lib/plausible/stats/filters/stats_api_filter_parser.ex +++ b/lib/plausible/stats/filters/stats_api_filter_parser.ex @@ -12,13 +12,12 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do Enum.map(filters, &parse_single_filter/1) |> Enum.reject(fn parsed -> parsed == :error end) - |> Enum.into(%{}) end defp parse_single_filter(str) do case to_kv(str) do ["event:goal" = key, raw_value] -> - {key, parse_goal_filter(raw_value)} + parse_goal_filter(key, raw_value) [key, raw_value] -> is_negated? = String.contains?(str, "!=") @@ -28,11 +27,11 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do final_value = remove_escape_chars(raw_value) cond do - is_wildcard? && is_negated? -> {key, {:does_not_match, raw_value}} - is_wildcard? -> {key, {:matches, raw_value}} - is_list? -> {key, {:member, parse_member_list(raw_value)}} - is_negated? -> {key, {:is_not, final_value}} - true -> {key, {:is, final_value}} + is_wildcard? && is_negated? -> [:does_not_match, key, raw_value] + is_wildcard? -> [:matches, key, raw_value] + is_list? -> [:member, key, parse_member_list(raw_value)] + is_negated? -> [:is_not, key, final_value] + true -> [:is, key, final_value] end |> reject_invalid_country_codes() @@ -41,7 +40,7 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do end end - defp reject_invalid_country_codes({"visit:country", {_, code_or_codes}} = filter) do + defp reject_invalid_country_codes([_op, "visit:country", code_or_codes] = filter) do code_or_codes |> List.wrap() |> Enum.reduce_while(filter, fn @@ -59,7 +58,7 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do |> Enum.map(&String.trim/1) end - defp parse_goal_filter(value) do + defp parse_goal_filter(key, value) do is_list? = list_expression?(value) is_wildcard? = wildcard_expression?(value) @@ -72,10 +71,10 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do |> wrap_goal_value() cond do - is_list? && is_wildcard? -> {:matches_member, value} - is_list? -> {:member, value} - is_wildcard? -> {:matches, value} - true -> {:is, value} + is_list? && is_wildcard? -> [:matches_member, key, value] + is_list? -> [:member, key, value] + is_wildcard? -> [:matches, key, value] + true -> [:is, key, value] end end end diff --git a/lib/plausible/stats/filters/where_builder.ex b/lib/plausible/stats/filters/where_builder.ex new file mode 100644 index 0000000000..04f66c8363 --- /dev/null +++ b/lib/plausible/stats/filters/where_builder.ex @@ -0,0 +1,304 @@ +defmodule Plausible.Stats.Filters.WhereBuilder do + @moduledoc """ + A module for building am ecto where clause of a query out of a query. + """ + + import Ecto.Query + import Plausible.Stats.Base, only: [page_regex: 1, utc_boundaries: 2] + + alias Plausible.Stats.Query + + use Plausible.Stats.Fragments + + @sessions_only_visit_fields [ + :entry_page, + :exit_page, + :entry_page_hostname, + :exit_page_hostname + ] + + @doc "Builds WHERE clause for a given Query against sessions or events table" + def build(table, site, query) do + base_condition = filter_site_time_range(table, site, query) + + query.filters + |> Enum.map(&add_filter(table, query, &1)) + |> Enum.reduce(base_condition, fn condition, acc -> dynamic([], ^acc and ^condition) end) + end + + @doc """ + Builds WHERE clause condition based off of a filter and a custom column name + Used for special business logic cases + + Accepts nil as the `filter` parameter, in which case the condition is a no-op (WHERE TRUE). + """ + def build_condition(db_field, filter) do + if filter do + filter_field(db_field, filter) + else + true + end + end + + defp filter_site_time_range(:events, site, query) do + {first_datetime, last_datetime} = utc_boundaries(query, site) + + dynamic( + [e], + e.site_id == ^site.id and e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime + ) + end + + defp filter_site_time_range(:sessions, site, %Query{experimental_session_count?: true} = query) do + {first_datetime, last_datetime} = utc_boundaries(query, site) + + # Counts each _active_ session in time range even if they started before + dynamic( + [s], + s.site_id == ^site.id and s.timestamp >= ^first_datetime and s.start < ^last_datetime + ) + end + + defp filter_site_time_range(:sessions, site, query) do + {first_datetime, last_datetime} = utc_boundaries(query, site) + + dynamic( + [s], + s.site_id == ^site.id and s.start >= ^first_datetime and s.start < ^last_datetime + ) + end + + defp add_filter(:events, _query, [:is, "event:name", name]) do + dynamic([e], e.name == ^name) + end + + defp add_filter(:events, _query, [:member, "event:name", list]) do + dynamic([e], e.name in ^list) + end + + defp add_filter(:events, _query, [:is, "event:goal", {:page, path}]) do + dynamic([e], e.pathname == ^path and e.name == "pageview") + end + + defp add_filter(:events, _query, [:matches, "event:goal", {:page, expr}]) do + regex = page_regex(expr) + + dynamic([e], fragment("match(?, ?)", e.pathname, ^regex) and e.name == "pageview") + end + + defp add_filter(:events, _query, [:is, "event:goal", {:event, event}]) do + dynamic([e], e.name == ^event) + end + + defp add_filter(:events, _query, [:member, "event:goal", clauses]) do + {events, pages} = split_goals(clauses) + + dynamic([e], (e.pathname in ^pages and e.name == "pageview") or e.name in ^events) + end + + defp add_filter(:events, _query, [:matches_member, "event:goal", clauses]) do + {events, pages} = split_goals(clauses, &page_regex/1) + + event_clause = + if Enum.any?(events) do + dynamic([x], fragment("multiMatchAny(?, ?)", x.name, ^events)) + else + dynamic([x], false) + end + + page_clause = + if Enum.any?(pages) do + dynamic( + [x], + fragment("multiMatchAny(?, ?)", x.pathname, ^pages) and x.name == "pageview" + ) + else + dynamic([x], false) + end + + where_clause = dynamic([], ^event_clause or ^page_clause) + + dynamic([e], ^where_clause) + end + + defp add_filter(:events, _query, [_, "event:page" | _rest] = filter) do + filter_field(:pathname, filter) + end + + defp add_filter(:events, _query, [_, "event:hostname" | _rest] = filter) do + filter_field(:hostname, filter) + end + + defp add_filter(:events, _query, [_, "event:props:" <> prop_name | _rest] = filter) do + filter_custom_prop(prop_name, :meta, filter) + end + + defp add_filter(:events, _query, [_, "visit:entry_props:" <> _prop_name | _rest]) do + true + end + + defp add_filter( + :events, + %Query{experimental_reduced_joins?: true}, + [_, "visit:" <> key | _rest] = filter + ) do + # Filter events query if experimental_reduced_joins? is true + field_name = String.to_existing_atom(key) + + if Enum.member?(@sessions_only_visit_fields, field_name) do + true + else + filter_field(field_name, filter) + end + end + + defp add_filter(:events, _query, [_, "visit:" <> _key | _rest]) do + true + end + + defp add_filter(:sessions, _query, [_, "visit:entry_props:" <> prop_name | _rest] = filter) do + filter_custom_prop(prop_name, :entry_meta, filter) + end + + defp add_filter(:sessions, _query, [_, "visit:" <> key | _rest] = filter) do + filter_field(String.to_existing_atom(key), filter) + end + + defp add_filter(:sessions, _query, [_, "event:" <> _ | _rest]) do + # Cannot apply sessions filters directly on session query where clause. + true + end + + defp filter_custom_prop(prop_name, column_name, [:is, _, "(none)"]) do + dynamic([t], not has_key(t, column_name, ^prop_name)) + end + + defp filter_custom_prop(prop_name, column_name, [:is, _, value]) do + dynamic( + [t], + has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) == ^value + ) + end + + defp filter_custom_prop(prop_name, column_name, [:is_not, _, "(none)"]) do + dynamic([t], has_key(t, column_name, ^prop_name)) + end + + defp filter_custom_prop(prop_name, column_name, [:is_not, _, value]) do + dynamic( + [t], + not has_key(t, column_name, ^prop_name) or get_by_key(t, column_name, ^prop_name) != ^value + ) + end + + defp filter_custom_prop(prop_name, column_name, [:matches, _, value]) do + regex = page_regex(value) + + dynamic( + [t], + has_key(t, column_name, ^prop_name) and + fragment("match(?, ?)", get_by_key(t, column_name, ^prop_name), ^regex) + ) + end + + defp filter_custom_prop(prop_name, column_name, [:member, _, values]) do + none_value_included = Enum.member?(values, "(none)") + + dynamic( + [t], + (has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) in ^values) or + (^none_value_included and not has_key(t, column_name, ^prop_name)) + ) + end + + defp filter_custom_prop(prop_name, column_name, [:not_member, _, values]) do + none_value_included = Enum.member?(values, "(none)") + + dynamic( + [t], + (has_key(t, column_name, ^prop_name) and + get_by_key(t, column_name, ^prop_name) not in ^values) or + (^none_value_included and + has_key(t, column_name, ^prop_name) and + get_by_key(t, column_name, ^prop_name) not in ^values) or + (not (^none_value_included) and not has_key(t, column_name, ^prop_name)) + ) + end + + defp filter_custom_prop(prop_name, column_name, [:matches_member, _, clauses]) do + regexes = Enum.map(clauses, &page_regex/1) + + dynamic( + [t], + has_key(t, column_name, ^prop_name) and + fragment( + "arrayExists(k -> match(?, k), ?)", + get_by_key(t, column_name, ^prop_name), + ^regexes + ) + ) + end + + defp filter_field(db_field, [:is, _key, value]) do + value = db_field_val(db_field, value) + dynamic([x], field(x, ^db_field) == ^value) + end + + defp filter_field(db_field, [:is_not, _key, value]) do + value = db_field_val(db_field, value) + dynamic([x], field(x, ^db_field) != ^value) + end + + defp filter_field(db_field, [:matches_member, _key, glob_exprs]) do + page_regexes = Enum.map(glob_exprs, &page_regex/1) + dynamic([x], fragment("multiMatchAny(?, ?)", field(x, ^db_field), ^page_regexes)) + end + + defp filter_field(db_field, [:not_matches_member, _key, glob_exprs]) do + page_regexes = Enum.map(glob_exprs, &page_regex/1) + dynamic([x], fragment("not(multiMatchAny(?, ?))", field(x, ^db_field), ^page_regexes)) + end + + defp filter_field(db_field, [:matches, _key, glob_expr]) do + regex = page_regex(glob_expr) + dynamic([x], fragment("match(?, ?)", field(x, ^db_field), ^regex)) + end + + defp filter_field(db_field, [:does_not_match, _key, glob_expr]) do + regex = page_regex(glob_expr) + dynamic([x], fragment("not(match(?, ?))", field(x, ^db_field), ^regex)) + end + + defp filter_field(db_field, [:member, _key, list]) do + list = Enum.map(list, &db_field_val(db_field, &1)) + dynamic([x], field(x, ^db_field) in ^list) + end + + defp filter_field(db_field, [:not_member, _key, list]) do + list = Enum.map(list, &db_field_val(db_field, &1)) + dynamic([x], field(x, ^db_field) not in ^list) + end + + @no_ref "Direct / None" + @not_set "(not set)" + + defp db_field_val(:source, @no_ref), do: "" + defp db_field_val(:referrer, @no_ref), do: "" + defp db_field_val(:utm_medium, @no_ref), do: "" + defp db_field_val(:utm_source, @no_ref), do: "" + defp db_field_val(:utm_campaign, @no_ref), do: "" + defp db_field_val(:utm_content, @no_ref), do: "" + defp db_field_val(:utm_term, @no_ref), do: "" + defp db_field_val(_, @not_set), do: "" + defp db_field_val(_, val), do: val + + defp split_goals(clauses, map_fn \\ &Function.identity/1) do + groups = + Enum.group_by(clauses, fn {goal_type, _v} -> goal_type end, fn {_k, val} -> map_fn.(val) end) + + { + Map.get(groups, :event, []), + Map.get(groups, :page, []) + } + end +end diff --git a/lib/plausible/stats/fragments.ex b/lib/plausible/stats/fragments.ex index ae5ef4e44c..27f49839f5 100644 --- a/lib/plausible/stats/fragments.ex +++ b/lib/plausible/stats/fragments.ex @@ -110,7 +110,7 @@ defmodule Plausible.Stats.Fragments do quote do fragment( "has(?, ?)", - field(unquote(table), unquote(meta_key_column(meta_column))), + field(unquote(table), ^meta_key_column(unquote(meta_column))), unquote(key) ) end @@ -130,18 +130,18 @@ defmodule Plausible.Stats.Fragments do quote do fragment( "?[indexOf(?, ?)]", - field(unquote(table), unquote(meta_value_column(meta_column))), - field(unquote(table), unquote(meta_key_column(meta_column))), + field(unquote(table), ^meta_value_column(unquote(meta_column))), + field(unquote(table), ^meta_key_column(unquote(meta_column))), unquote(key) ) end end - defp meta_key_column(:meta), do: :"meta.key" - defp meta_key_column(:entry_meta), do: :"entry_meta.key" + def meta_key_column(:meta), do: :"meta.key" + def meta_key_column(:entry_meta), do: :"entry_meta.key" - defp meta_value_column(:meta), do: :"meta.value" - defp meta_value_column(:entry_meta), do: :"entry_meta.value" + def meta_value_column(:meta), do: :"meta.value" + def meta_value_column(:entry_meta), do: :"entry_meta.value" defmacro __using__(_) do quote do diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index 9ab3dd703e..3c43c17f85 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -33,27 +33,29 @@ defmodule Plausible.Stats.Imported do @imported_properties Map.keys(@property_to_table_mappings) + @goals_with_url ["Outbound Link: Click", "Cloaked Link: Click", "File Download"] + + @doc """ + Returns a boolean indicating whether the combination of filters and + breakdown property is possible to query from the imported tables. + + Usually, when no filters are used, the imported schema supports the + query. There is one exception though - breakdown by a custom property. + We are currently importing only two custom properties - `url` and `path. + Both these properties can only be used with their special goal filter + (see `@goals_with_url` and `@goals_with_path`). + """ def schema_supports_query?(query) do - filter_count = length(Map.keys(query.filters)) + filter_count = length(query.filters) case {filter_count, query.property} do {0, "event:props:" <> _} -> false {0, _} -> true - {1, _} -> supports_single_filter?(query) + {1, "event:props:url"} -> has_special_goal_filter?(query, @goals_with_url) {_, _} -> false end end - defp supports_single_filter?(%Query{ - filters: %{"event:goal" => {:is, {:event, event}}}, - property: "event:props:url" - }) - when event in ["Outbound Link: Click", "File Download"] do - true - end - - defp supports_single_filter?(_query), do: false - def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _), do: native_q @@ -118,7 +120,7 @@ defmodule Plausible.Stats.Imported do where: i.visitors > 0, select: %{} ) - |> maybe_apply_filter(query.filters, property, dim) + |> maybe_apply_filter(query, property, dim) |> group_imported_by(dim) |> select_imported_metrics(metrics) @@ -214,23 +216,35 @@ defmodule Plausible.Stats.Imported do ) end - defp maybe_apply_filter( - q, - %{"event:goal" => {:is, {:event, event_name}}}, - "event:props:url", - _dim - ) - when event_name in ["Outbound Link: Click", "File Download"] do - where(q, [i], i.name == ^event_name) + defp maybe_apply_filter(q, query, "event:props:url", _) do + if name = find_special_goal_filter(query, @goals_with_url) do + where(q, [i], i.name == ^name) + else + q + end end - defp maybe_apply_filter(q, filters, property, dim) do - case filters[property] do - {:member, list} -> where(q, [i], field(i, ^dim) in ^list) + defp maybe_apply_filter(q, query, property, dim) do + case Query.get_filter(query, property) do + [:member, _, list] -> where(q, [i], field(i, ^dim) in ^list) _ -> q end end + defp has_special_goal_filter?(query, event_names) do + not is_nil(find_special_goal_filter(query, event_names)) + end + + defp find_special_goal_filter(query, event_names) do + case Query.get_filter(query, "event:goal") do + [:is, "event:goal", {:event, name}] -> + if name in event_names, do: name, else: nil + + _ -> + nil + end + end + defp select_imported_metrics(q, []), do: q defp select_imported_metrics(q, [:visitors | rest]) do diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 5af5ede7f4..4fa8f4a94f 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -5,7 +5,7 @@ defmodule Plausible.Stats.Query do interval: nil, period: nil, property: nil, - filters: %{}, + filters: [], sample_threshold: 20_000_000, imported_data_requested: false, include_imported: false, @@ -200,50 +200,36 @@ defmodule Plausible.Stats.Query do struct!(query, filters: Filters.parse(params["filters"])) end - def put_filter(query, key, val) do - parsed_val = - if is_binary(val) do - Filters.DashboardFilterParser.filter_value(key, val) - else - val - end - + def put_filter(query, filter) do struct!(query, - filters: Map.put(query.filters, key, parsed_val) + filters: query.filters ++ [filter] ) end - def remove_event_filters(query, opts) do + def remove_filters(query, prefixes) do new_filters = - Enum.filter(query.filters, fn {filter_key, _} -> - cond do - :page in opts && filter_key == "event:page" -> false - :goal in opts && filter_key == "event:goal" -> false - :props in opts && filter_key && String.starts_with?(filter_key, "event:props:") -> false - true -> true - end + Enum.reject(query.filters, fn [_, filter_key | _rest] -> + Enum.any?(prefixes, &String.starts_with?(filter_key, &1)) end) - |> Enum.into(%{}) struct!(query, filters: new_filters) end def has_event_filters?(query) do - Enum.any?(query.filters, fn - {"event:" <> _, _} -> true - _ -> false + Enum.any?(query.filters, fn [_op, prop | _rest] -> + String.starts_with?(prop, "event:") end) end def get_filter_by_prefix(query, prefix) do - Enum.find(query.filters, fn {prop, _value} -> + Enum.find(query.filters, fn [_op, prop | _rest] -> String.starts_with?(prop, prefix) end) end - def get_all_filters_by_prefix(query, prefix) do - Enum.filter(query.filters, fn {prop, _value} -> - String.starts_with?(prop, prefix) + def get_filter(query, name) do + Enum.find(query.filters, fn [_op, prop | _rest] -> + prop == name end) end @@ -281,7 +267,12 @@ defmodule Plausible.Stats.Query do @spec trace(%__MODULE__{}, [atom()]) :: %__MODULE__{} def trace(%__MODULE__{} = query, metrics) do - filter_keys = Map.keys(query.filters) |> Enum.sort() |> Enum.join(";") + filter_keys = + query.filters + |> Enum.map(fn [_op, prop | _rest] -> prop end) + |> Enum.sort() + |> Enum.join(";") + metrics = metrics |> Enum.sort() |> Enum.join(";") Tracer.set_attributes([ diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.ex index ef976a0604..8b77ca5928 100644 --- a/lib/plausible/stats/table_decider.ex +++ b/lib/plausible/stats/table_decider.ex @@ -23,15 +23,15 @@ defmodule Plausible.Stats.TableDecider do partition(metrics, query, &metric_partitioner/2) # Treat breakdown property as yet another filter - filters = + query = if breakdown_property do - Map.put(query.filters, breakdown_property, nil) + Query.put_filter(query, [:is, breakdown_property, []]) else - query.filters + query end %{event: event_only_filters, session: session_only_filters} = - partition(filters, query, &filters_partitioner/2) + partition(query.filters, query, &filters_partitioner/2) cond do # Only one table needs to be queried @@ -83,16 +83,16 @@ defmodule Plausible.Stats.TableDecider do defp metric_partitioner(_, _), do: :either - defp filters_partitioner(_, {"event:" <> _, _}), do: :event - defp filters_partitioner(_, {"visit:entry_page", _}), do: :session - defp filters_partitioner(_, {"visit:entry_page_hostname", _}), do: :session - defp filters_partitioner(_, {"visit:exit_page", _}), do: :session - defp filters_partitioner(_, {"visit:exit_page_hostname", _}), do: :session + defp filters_partitioner(_, [_, "event:" <> _ | _rest]), do: :event + defp filters_partitioner(_, [_, "visit:entry_page" | _rest]), do: :session + defp filters_partitioner(_, [_, "visit:entry_page_hostname" | _rest]), do: :session + defp filters_partitioner(_, [_, "visit:exit_page" | _rest]), do: :session + defp filters_partitioner(_, [_, "visit:exit_page_hostname" | _rest]), do: :session - defp filters_partitioner(%Query{experimental_reduced_joins?: true}, {"visit:" <> _, _}), + defp filters_partitioner(%Query{experimental_reduced_joins?: true}, [_, "visit:" <> _ | _rest]), do: :either - defp filters_partitioner(_, {"visit:" <> _, _}), + defp filters_partitioner(_, [_, "visit:" <> _ | _rest]), do: :session defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) do diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 0ca145330b..08dfaf18bf 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -314,7 +314,7 @@ defmodule Plausible.Stats.Timeseries do defp maybe_add_timeseries_conversion_rate(q, site, query, metrics) do if :conversion_rate in metrics do - totals_query = query |> Query.remove_event_filters([:goal, :props]) + totals_query = query |> Query.remove_filters(["event:goal", "event:props"]) totals_timeseries_q = from(e in base_event_query(site, totals_query), diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index 0623873993..a14abf6cd7 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -119,7 +119,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do {_, _, :all} -> true - {{"event:props:" <> prop, _}, _property, allowed_props} -> + {[_, "event:props:" <> prop | _], _property, allowed_props} -> prop in allowed_props {_filter, "event:props:" <> prop, allowed_props} -> @@ -156,10 +156,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController do defp validate_metric("time_on_page" = metric, query) do cond do - query.filters["event:goal"] -> + Query.get_filter(query, "event:goal") -> {:error, "Metric `#{metric}` cannot be queried when filtering by `event:goal`"} - query.filters["event:name"] -> + Query.get_filter(query, "event:name") -> {:error, "Metric `#{metric}` cannot be queried when filtering by `event:name`"} query.property == "event:page" -> @@ -169,7 +169,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do {:error, "Metric `#{metric}` is not supported in breakdown queries (except `event:page` breakdown)"} - query.filters["event:page"] -> + Query.get_filter(query, "event:page") -> {:ok, metric} true -> @@ -183,7 +183,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do query.property == "event:goal" -> {:ok, metric} - query.filters["event:goal"] -> + Query.get_filter(query, "event:goal") -> {:ok, metric} true -> @@ -198,7 +198,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do defp validate_metric("views_per_visit" = metric, query) do cond do - query.filters["event:page"] -> + Query.get_filter(query, "event:page") -> {:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."} query.property != nil -> @@ -235,7 +235,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController do end defp find_event_only_filter(query) do - Map.keys(query.filters) |> Enum.find(&event_only_property?/1) + query.filters + |> Enum.map(fn [_op, prop | _] -> prop end) + |> Enum.find(&event_only_property?/1) end defp event_only_property?("event:name"), do: true @@ -327,7 +329,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do end) end - defp validate_filter(site, {"event:goal", {_type, goal_filter}}) do + defp validate_filter(site, [_type, "event:goal", goal_filter]) do configured_goals = Plausible.Goals.for_site(site) |> Enum.map(fn @@ -350,7 +352,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do end end - defp validate_filter(_site, {property, _}) do + defp validate_filter(_site, [_, property | _]) do if Plausible.Stats.Props.valid_prop?(property) do :ok else diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index dc2a562cdd..bc9086ca84 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -6,6 +6,7 @@ defmodule PlausibleWeb.Api.StatsController do alias Plausible.Stats alias Plausible.Stats.{Query, Comparisons} + alias Plausible.Stats.Filters.DashboardFilterParser alias PlausibleWeb.Api.Helpers, as: H require Logger @@ -276,11 +277,25 @@ defmodule PlausibleWeb.Api.StatsController do end end - defp fetch_top_stats( - site, - %Query{period: "realtime", filters: %{"event:goal" => _goal}} = query, - _comparison_query - ) do + defp fetch_top_stats(site, query, comparison_query) do + goal_filter = Query.get_filter(query, "event:goal") + + cond do + query.period == "realtime" && goal_filter -> + fetch_goal_realtime_top_stats(site, query, comparison_query) + + query.period == "realtime" -> + fetch_realtime_top_stats(site, query, comparison_query) + + goal_filter -> + fetch_goal_top_stats(site, query, comparison_query) + + true -> + fetch_other_top_stats(site, query, comparison_query) + end + end + + defp fetch_goal_realtime_top_stats(site, query, _comparison_query) do query_30m = %Query{query | period: "30m"} %{ @@ -308,7 +323,7 @@ defmodule PlausibleWeb.Api.StatsController do {stats, 100} end - defp fetch_top_stats(site, %Query{period: "realtime"} = query, _comparison_query) do + defp fetch_realtime_top_stats(site, query, _comparison_query) do query_30m = %Query{query | period: "30m"} %{ @@ -336,7 +351,7 @@ defmodule PlausibleWeb.Api.StatsController do {stats, 100} end - defp fetch_top_stats(site, %Query{filters: %{"event:goal" => _}} = query, comparison_query) do + defp fetch_goal_top_stats(site, query, comparison_query) do metrics = [:total_visitors, :visitors, :events, :conversion_rate] ++ @revenue_metrics @@ -365,9 +380,9 @@ defmodule PlausibleWeb.Api.StatsController do |> then(&{&1, 100}) end - defp fetch_top_stats(site, query, comparison_query) do + defp fetch_other_top_stats(site, query, comparison_query) do metrics = - if query.filters["event:page"] do + if Query.get_filter(query, "event:page") do [ :visitors, :visits, @@ -469,7 +484,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{source: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do res |> transform_keys(%{visitors: :conversions}) |> to_csv([:name, :conversions, :conversion_rate]) @@ -519,17 +534,17 @@ defmodule PlausibleWeb.Api.StatsController do end defp validate_funnel_query(query) do - case query do - _ when is_map_key(query.filters, "event:goal") -> + cond do + Query.get_filter(query, "event:goal") -> {:error, {:invalid_funnel_query, "goals"}} - _ when is_map_key(query.filters, "event:page") -> + Query.get_filter(query, "event:page") -> {:error, {:invalid_funnel_query, "pages"}} - _ when query.period == "realtime" -> + query.period == "realtime" -> {:error, {:invalid_funnel_query, "realtime period"}} - _ -> + true -> :ok end end @@ -547,7 +562,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{utm_medium: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do res |> transform_keys(%{visitors: :conversions}) |> to_csv([:name, :conversions, :conversion_rate]) @@ -571,7 +586,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{utm_campaign: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do res |> transform_keys(%{visitors: :conversions}) |> to_csv([:name, :conversions, :conversion_rate]) @@ -595,7 +610,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{utm_content: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do res |> transform_keys(%{visitors: :conversions}) |> to_csv([:name, :conversions, :conversion_rate]) @@ -619,7 +634,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{utm_term: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do res |> transform_keys(%{visitors: :conversions}) |> to_csv([:name, :conversions, :conversion_rate]) @@ -643,7 +658,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{utm_source: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do res |> transform_keys(%{visitors: :conversions}) |> to_csv([:name, :conversions, :conversion_rate]) @@ -667,7 +682,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{referrer: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do res |> transform_keys(%{visitors: :conversions}) |> to_csv([:name, :conversions, :conversion_rate]) @@ -711,9 +726,11 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] params = Map.put(params, "property", "visit:referrer") + referrer_filter = DashboardFilterParser.filter_value("visit:source", referrer) + query = Query.from(site, params) - |> Query.put_filter("visit:source", referrer) + |> Query.put_filter(referrer_filter) pagination = parse_pagination(params) @@ -747,7 +764,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{page: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do pages |> transform_keys(%{visitors: :conversions}) |> to_csv([:name, :conversions, :conversion_rate]) @@ -771,7 +788,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{entry_page: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do to_csv(entry_pages, [:name, :visitors, :conversion_rate], [ :name, :conversions, @@ -803,7 +820,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{exit_page: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do to_csv(exit_pages, [:name, :visitors, :conversion_rate], [ :name, :conversions, @@ -829,8 +846,9 @@ defmodule PlausibleWeb.Api.StatsController do pages = Enum.map(breakdown_results, & &1[:exit_page]) total_visits_query = - Query.put_filter(query, "event:page", {:member, pages}) - |> Query.put_filter("event:name", {:is, "pageview"}) + query + |> Query.put_filter([:member, "event:page", pages]) + |> Query.put_filter([:is, "event:name", "pageview"]) |> struct!(property: "event:page") total_pageviews = @@ -870,7 +888,7 @@ defmodule PlausibleWeb.Api.StatsController do Map.put(country, :name, country_info.name) end) - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do countries |> transform_keys(%{visitors: :conversions}) |> to_csv([:name, :conversions, :conversion_rate]) @@ -926,7 +944,7 @@ defmodule PlausibleWeb.Api.StatsController do end) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do regions |> transform_keys(%{visitors: :conversions}) |> to_csv([:name, :conversions, :conversion_rate]) @@ -966,7 +984,7 @@ defmodule PlausibleWeb.Api.StatsController do end) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do cities |> transform_keys(%{visitors: :conversions}) |> to_csv([:name, :conversions, :conversion_rate]) @@ -990,7 +1008,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{browser: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do browsers |> transform_keys(%{visitors: :conversions}) |> to_csv([:name, :conversions, :conversion_rate]) @@ -1014,7 +1032,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{browser_version: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do versions |> transform_keys(%{ name: :version, @@ -1044,7 +1062,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{os: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do systems |> transform_keys(%{visitors: :conversions}) |> to_csv([:name, :conversions, :conversion_rate]) @@ -1068,7 +1086,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{os_version: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do versions |> transform_keys(%{name: :version, os: :name, visitors: :conversions}) |> to_csv([:name, :version, :conversions, :conversion_rate]) @@ -1094,7 +1112,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{device: :name}) if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do + if Query.get_filter(query, "event:goal") do sizes |> transform_keys(%{visitors: :conversions}) |> to_csv([:name, :conversions, :conversion_rate]) @@ -1183,7 +1201,7 @@ defmodule PlausibleWeb.Api.StatsController do |> Enum.concat() percent_or_cr = - if query.filters["event:goal"], + if Query.get_filter(query, "event:goal"), do: :conversion_rate, else: :percentage @@ -1200,7 +1218,7 @@ defmodule PlausibleWeb.Api.StatsController do query = Query.from(site, params) metrics = - if query.filters["event:goal"] do + if Query.get_filter(query, "event:goal") do [:visitors, :events, :conversion_rate] ++ @revenue_metrics else [:visitors, :events, :percentage] ++ @revenue_metrics @@ -1359,7 +1377,7 @@ defmodule PlausibleWeb.Api.StatsController do requires_goal_filter? = metric in [:conversion_rate, :events] - if requires_goal_filter? and !query.filters["event:goal"] do + if requires_goal_filter? and !Query.get_filter(query, "event:goal") do {:error, "Metric `#{metric}` can only be queried with a goal filter"} else {:ok, metric} @@ -1401,7 +1419,7 @@ defmodule PlausibleWeb.Api.StatsController do end defp breakdown_metrics(query, extra_metrics \\ []) do - if query.filters["event:goal"] do + if Query.get_filter(query, "event:goal") do [:visitors, :conversion_rate, :total_visitors] else [:visitors] ++ extra_metrics diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index ca921c045f..d198647432 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -180,16 +180,28 @@ defmodule PlausibleWeb.StatsController do |> Enum.join() end - defp csv_graph_metrics(%Query{filters: %{"event:goal" => _}}) do - metrics = [:visitors, :events, :conversion_rate] - column_headers = [:date, :unique_conversions, :total_conversions, :conversion_rate] + defp csv_graph_metrics(query) do + {metrics, column_headers} = + if Query.get_filter(query, "event:goal") do + { + [:visitors, :events, :conversion_rate], + [:date, :unique_conversions, :total_conversions, :conversion_rate] + } + else + metrics = [ + :visitors, + :pageviews, + :visits, + :views_per_visit, + :bounce_rate, + :visit_duration + ] - {metrics, column_headers} - end - - defp csv_graph_metrics(_) do - metrics = [:visitors, :pageviews, :visits, :views_per_visit, :bounce_rate, :visit_duration] - column_headers = [:date | metrics] + { + metrics, + [:date | metrics] + } + end {metrics, column_headers} end diff --git a/test/plausible/google/search_console/filters_test.exs b/test/plausible/google/search_console/filters_test.exs index 7d73dacda8..c6efffa069 100644 --- a/test/plausible/google/search_console/filters_test.exs +++ b/test/plausible/google/search_console/filters_test.exs @@ -3,9 +3,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do use Plausible.DataCase, async: true test "transforms simple page filter" do - filters = %{ - "visit:entry_page" => {:is, "/page"} - } + filters = [ + [:is, "visit:entry_page", "/page"] + ] {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) @@ -15,9 +15,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do end test "transforms matches page filter" do - filters = %{ - "visit:entry_page" => {:matches, "*page*"} - } + filters = [ + [:matches, "visit:entry_page", "*page*"] + ] {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) @@ -35,9 +35,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do end test "transforms member page filter" do - filters = %{ - "visit:entry_page" => {:member, ["/pageA", "/pageB"]} - } + filters = [ + [:member, "visit:entry_page", ["/pageA", "/pageB"]] + ] {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) @@ -55,9 +55,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do end test "transforms matches_member page filter" do - filters = %{ - "visit:entry_page" => {:matches_member, ["/pageA*", "/pageB*"]} - } + filters = [ + [:matches_member, "visit:entry_page", ["/pageA*", "/pageB*"]] + ] {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) @@ -75,9 +75,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do end test "transforms event:page exactly like visit:entry_page" do - filters = %{ - "event:page" => {:matches_member, ["/pageA*", "/pageB*"]} - } + filters = [ + [:matches_member, "event:page", ["/pageA*", "/pageB*"]] + ] {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) @@ -95,9 +95,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do end test "transforms simple visit:screen filter" do - filters = %{ - "visit:screen" => {:is, "Desktop"} - } + filters = [ + [:is, "visit:screen", "Desktop"] + ] {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) @@ -105,9 +105,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do end test "transforms member visit:screen filter" do - filters = %{ - "visit:screen" => {:member, ["Mobile", "Tablet"]} - } + filters = [ + [:member, "visit:screen", ["Mobile", "Tablet"]] + ] {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) @@ -121,9 +121,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do end test "transforms simple visit:country filter to alpha3" do - filters = %{ - "visit:country" => {:is, "EE"} - } + filters = [ + [:is, "visit:country", "EE"] + ] {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) @@ -131,9 +131,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do end test "transforms member visit:country filter" do - filters = %{ - "visit:country" => {:member, ["EE", "PL"]} - } + filters = [ + [:member, "visit:country", ["EE", "PL"]] + ] {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) @@ -147,11 +147,11 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do end test "filters can be combined" do - filters = %{ - "visit:entry_page" => {:matches, "*web-analytics*"}, - "visit:screen" => {:is, "Desktop"}, - "visit:country" => {:member, ["EE", "PL"]} - } + filters = [ + [:member, "visit:country", ["EE", "PL"]], + [:matches, "visit:entry_page", "*web-analytics*"], + [:is, "visit:screen", "Desktop"] + ] {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters) @@ -171,12 +171,12 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do end test "when unsupported filter is included the whole set becomes invalid" do - filters = %{ - "visit:entry_page" => {:matches, "*web-analytics*"}, - "visit:screen" => {:is, "Desktop"}, - "visit:country" => {:member, ["EE", "PL"]}, - "visit:utm_medium" => {:is, "facebook"} - } + filters = [ + [:matches, "visit:entry_page", "*web-analytics*"], + [:is, "visit:screen", "Desktop"], + [:member, "visit:country", ["EE", "PL"]], + [:is, "visit:utm_medium", "facebook"] + ] assert :unsupported_filters = Filters.transform("sc-domain:plausible.io", filters) end diff --git a/test/plausible/stats/dashboard_filter_parser_test.exs b/test/plausible/stats/dashboard_filter_parser_test.exs index e5469bdfda..1602380f37 100644 --- a/test/plausible/stats/dashboard_filter_parser_test.exs +++ b/test/plausible/stats/dashboard_filter_parser_test.exs @@ -9,199 +9,199 @@ defmodule Plausible.Stats.DashboardFilterParserTest do describe "adding prefix" do test "adds appropriate prefix to filter" do %{"page" => "/"} - |> assert_parsed(%{"event:page" => {:is, "/"}}) + |> assert_parsed([[:is, "event:page", "/"]]) %{"goal" => "Signup"} - |> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}}) + |> assert_parsed([[:is, "event:goal", {:event, "Signup"}]]) %{"goal" => "Visit /blog"} - |> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}}) + |> assert_parsed([[:is, "event:goal", {:page, "/blog"}]]) %{"source" => "Google"} - |> assert_parsed(%{"visit:source" => {:is, "Google"}}) + |> assert_parsed([[:is, "visit:source", "Google"]]) %{"referrer" => "cnn.com"} - |> assert_parsed(%{"visit:referrer" => {:is, "cnn.com"}}) + |> assert_parsed([[:is, "visit:referrer", "cnn.com"]]) %{"utm_medium" => "search"} - |> assert_parsed(%{"visit:utm_medium" => {:is, "search"}}) + |> assert_parsed([[:is, "visit:utm_medium", "search"]]) %{"utm_source" => "bing"} - |> assert_parsed(%{"visit:utm_source" => {:is, "bing"}}) + |> assert_parsed([[:is, "visit:utm_source", "bing"]]) %{"utm_content" => "content"} - |> assert_parsed(%{"visit:utm_content" => {:is, "content"}}) + |> assert_parsed([[:is, "visit:utm_content", "content"]]) %{"utm_term" => "term"} - |> assert_parsed(%{"visit:utm_term" => {:is, "term"}}) + |> assert_parsed([[:is, "visit:utm_term", "term"]]) %{"screen" => "Desktop"} - |> assert_parsed(%{"visit:screen" => {:is, "Desktop"}}) + |> assert_parsed([[:is, "visit:screen", "Desktop"]]) %{"browser" => "Opera"} - |> assert_parsed(%{"visit:browser" => {:is, "Opera"}}) + |> assert_parsed([[:is, "visit:browser", "Opera"]]) %{"browser_version" => "10.1"} - |> assert_parsed(%{"visit:browser_version" => {:is, "10.1"}}) + |> assert_parsed([[:is, "visit:browser_version", "10.1"]]) %{"os" => "Linux"} - |> assert_parsed(%{"visit:os" => {:is, "Linux"}}) + |> assert_parsed([[:is, "visit:os", "Linux"]]) %{"os_version" => "13.0"} - |> assert_parsed(%{"visit:os_version" => {:is, "13.0"}}) + |> assert_parsed([[:is, "visit:os_version", "13.0"]]) %{"country" => "EE"} - |> assert_parsed(%{"visit:country" => {:is, "EE"}}) + |> assert_parsed([[:is, "visit:country", "EE"]]) %{"region" => "EE-12"} - |> assert_parsed(%{"visit:region" => {:is, "EE-12"}}) + |> assert_parsed([[:is, "visit:region", "EE-12"]]) %{"city" => "123"} - |> assert_parsed(%{"visit:city" => {:is, "123"}}) + |> assert_parsed([[:is, "visit:city", "123"]]) %{"entry_page" => "/blog"} - |> assert_parsed(%{"visit:entry_page" => {:is, "/blog"}}) + |> assert_parsed([[:is, "visit:entry_page", "/blog"]]) %{"exit_page" => "/blog"} - |> assert_parsed(%{"visit:exit_page" => {:is, "/blog"}}) + |> assert_parsed([[:is, "visit:exit_page", "/blog"]]) %{"props" => %{"cta" => "Top"}} - |> assert_parsed(%{"event:props:cta" => {:is, "Top"}}) + |> assert_parsed([[:is, "event:props:cta", "Top"]]) %{"hostname" => "dummy.site"} - |> assert_parsed(%{"event:hostname" => {:is, "dummy.site"}}) + |> assert_parsed([[:is, "event:hostname", "dummy.site"]]) end end describe "escaping pipe character" do test "in simple is filter" do %{"goal" => ~S(Foo \| Bar)} - |> assert_parsed(%{"event:goal" => {:is, {:event, "Foo | Bar"}}}) + |> assert_parsed([[:is, "event:goal", {:event, "Foo | Bar"}]]) end test "in member filter" do %{"page" => ~S(/|\|)} - |> assert_parsed(%{"event:page" => {:member, ["/", "|"]}}) + |> assert_parsed([[:member, "event:page", ["/", "|"]]]) end end describe "is not filter type" do test "simple is not filter" do %{"page" => "!/"} - |> assert_parsed(%{"event:page" => {:is_not, "/"}}) + |> assert_parsed([[:is_not, "event:page", "/"]]) %{"props" => %{"cta" => "!Top"}} - |> assert_parsed(%{"event:props:cta" => {:is_not, "Top"}}) + |> assert_parsed([[:is_not, "event:props:cta", "Top"]]) end end describe "member filter type" do test "simple member filter" do %{"page" => "/|/blog"} - |> assert_parsed(%{"event:page" => {:member, ["/", "/blog"]}}) + |> assert_parsed([[:member, "event:page", ["/", "/blog"]]]) end test "escaping pipe character" do %{"page" => "/|\\|"} - |> assert_parsed(%{"event:page" => {:member, ["/", "|"]}}) + |> assert_parsed([[:member, "event:page", ["/", "|"]]]) end test "mixed goals" do %{"goal" => "Signup|Visit /thank-you"} - |> assert_parsed(%{"event:goal" => {:member, [{:event, "Signup"}, {:page, "/thank-you"}]}}) + |> assert_parsed([[:member, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]]) %{"goal" => "Visit /thank-you|Signup"} - |> assert_parsed(%{"event:goal" => {:member, [{:page, "/thank-you"}, {:event, "Signup"}]}}) + |> assert_parsed([[:member, "event:goal", [{:page, "/thank-you"}, {:event, "Signup"}]]]) end end describe "matches_member filter type" do test "parses matches_member filter type" do %{"page" => "/|/blog**"} - |> assert_parsed(%{"event:page" => {:matches_member, ["/", "/blog**"]}}) + |> assert_parsed([[:matches_member, "event:page", ["/", "/blog**"]]]) end test "parses not_matches_member filter type" do %{"page" => "!/|/blog**"} - |> assert_parsed(%{"event:page" => {:not_matches_member, ["/", "/blog**"]}}) + |> assert_parsed([[:not_matches_member, "event:page", ["/", "/blog**"]]]) end end describe "contains filter type" do test "single contains" do %{"page" => "~blog"} - |> assert_parsed(%{"event:page" => {:matches, "**blog**"}}) + |> assert_parsed([[:matches, "event:page", "**blog**"]]) end test "negated contains" do %{"page" => "!~articles"} - |> assert_parsed(%{"event:page" => {:does_not_match, "**articles**"}}) + |> assert_parsed([[:does_not_match, "event:page", "**articles**"]]) end test "contains member" do %{"page" => "~articles|blog"} - |> assert_parsed(%{"event:page" => {:matches_member, ["**articles**", "**blog**"]}}) + |> assert_parsed([[:matches_member, "event:page", ["**articles**", "**blog**"]]]) end test "not contains member" do %{"page" => "!~articles|blog"} - |> assert_parsed(%{"event:page" => {:not_matches_member, ["**articles**", "**blog**"]}}) + |> assert_parsed([[:not_matches_member, "event:page", ["**articles**", "**blog**"]]]) end end describe "not_member filter type" do test "simple not_member filter" do %{"page" => "!/|/blog"} - |> assert_parsed(%{"event:page" => {:not_member, ["/", "/blog"]}}) + |> assert_parsed([[:not_member, "event:page", ["/", "/blog"]]]) end test "mixed goals" do %{"goal" => "!Signup|Visit /thank-you"} - |> assert_parsed(%{ - "event:goal" => {:not_member, [{:event, "Signup"}, {:page, "/thank-you"}]} - }) + |> assert_parsed([ + [:not_member, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]] + ]) %{"goal" => "!Visit /thank-you|Signup"} - |> assert_parsed(%{ - "event:goal" => {:not_member, [{:page, "/thank-you"}, {:event, "Signup"}]} - }) + |> assert_parsed([ + [:not_member, "event:goal", [{:page, "/thank-you"}, {:event, "Signup"}]] + ]) end end describe "matches filter type" do test "can be used with `goal` or `page` filters" do %{"page" => "/blog/post-*"} - |> assert_parsed(%{"event:page" => {:matches, "/blog/post-*"}}) + |> assert_parsed([[:matches, "event:page", "/blog/post-*"]]) %{"goal" => "Visit /blog/post-*"} - |> assert_parsed(%{"event:goal" => {:matches, {:page, "/blog/post-*"}}}) + |> assert_parsed([[:matches, "event:goal", {:page, "/blog/post-*"}]]) end test "other filters default to `is` even when wildcard is present" do %{"country" => "Germa**"} - |> assert_parsed(%{"visit:country" => {:is, "Germa**"}}) + |> assert_parsed([[:is, "visit:country", "Germa**"]]) end end describe "does_not_match filter type" do test "can be used with `page` filter" do %{"page" => "!/blog/post-*"} - |> assert_parsed(%{"event:page" => {:does_not_match, "/blog/post-*"}}) + |> assert_parsed([[:does_not_match, "event:page", "/blog/post-*"]]) end test "other filters default to is_not even when wildcard is present" do %{"country" => "!Germa**"} - |> assert_parsed(%{"visit:country" => {:is_not, "Germa**"}}) + |> assert_parsed([[:is_not, "visit:country", "Germa**"]]) end end describe "contains prefix filter type" do test "can be used with any filter" do %{"page" => "~/blog/post"} - |> assert_parsed(%{"event:page" => {:matches, "**/blog/post**"}}) + |> assert_parsed([[:matches, "event:page", "**/blog/post**"]]) %{"source" => "~facebook"} - |> assert_parsed(%{"visit:source" => {:matches, "**facebook**"}}) + |> assert_parsed([[:matches, "visit:source", "**facebook**"]]) end end end diff --git a/test/plausible/stats/filters_test.exs b/test/plausible/stats/filters_test.exs index 4f87bd9957..aa5b77412f 100644 --- a/test/plausible/stats/filters_test.exs +++ b/test/plausible/stats/filters_test.exs @@ -12,90 +12,90 @@ defmodule Plausible.Stats.FiltersTest do describe "parses filter expression" do test "simple positive" do "event:name==pageview" - |> assert_parsed(%{"event:name" => {:is, "pageview"}}) + |> assert_parsed([[:is, "event:name", "pageview"]]) end test "simple negative" do "event:name!=pageview" - |> assert_parsed(%{"event:name" => {:is_not, "pageview"}}) + |> assert_parsed([[:is_not, "event:name", "pageview"]]) end test "whitespace is trimmed" do " event:name == pageview " - |> assert_parsed(%{"event:name" => {:is, "pageview"}}) + |> assert_parsed([[:is, "event:name", "pageview"]]) end test "wildcard" do "event:page==/blog/post-*" - |> assert_parsed(%{"event:page" => {:matches, "/blog/post-*"}}) + |> assert_parsed([[:matches, "event:page", "/blog/post-*"]]) end test "negative wildcard" do "event:page!=/blog/post-*" - |> assert_parsed(%{"event:page" => {:does_not_match, "/blog/post-*"}}) + |> assert_parsed([[:does_not_match, "event:page", "/blog/post-*"]]) end test "custom event goal" do "event:goal==Signup" - |> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}}) + |> assert_parsed([[:is, "event:goal", {:event, "Signup"}]]) end test "pageview goal" do "event:goal==Visit /blog" - |> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}}) + |> assert_parsed([[:is, "event:goal", {:page, "/blog"}]]) end test "member" do "visit:country==FR|GB|DE" - |> assert_parsed(%{"visit:country" => {:member, ["FR", "GB", "DE"]}}) + |> assert_parsed([[:member, "visit:country", ["FR", "GB", "DE"]]]) end test "member + wildcard" do "event:page==/blog**|/newsletter|/*/" - |> assert_parsed(%{"event:page" => {:matches, "/blog**|/newsletter|/*/"}}) + |> assert_parsed([[:matches, "event:page", "/blog**|/newsletter|/*/"]]) end test "combined with \";\"" do "event:page==/blog**|/newsletter|/*/ ; visit:country==FR|GB|DE" - |> assert_parsed(%{ - "event:page" => {:matches, "/blog**|/newsletter|/*/"}, - "visit:country" => {:member, ["FR", "GB", "DE"]} - }) + |> assert_parsed([ + [:matches, "event:page", "/blog**|/newsletter|/*/"], + [:member, "visit:country", ["FR", "GB", "DE"]] + ]) end test "escaping pipe character" do "utm_campaign==campaign \\| 1" - |> assert_parsed(%{"utm_campaign" => {:is, "campaign | 1"}}) + |> assert_parsed([[:is, "utm_campaign", "campaign | 1"]]) end test "escaping pipe character in member filter" do "utm_campaign==campaign \\| 1|campaign \\| 2" - |> assert_parsed(%{"utm_campaign" => {:member, ["campaign | 1", "campaign | 2"]}}) + |> assert_parsed([[:member, "utm_campaign", ["campaign | 1", "campaign | 2"]]]) end test "keeps escape characters in member + wildcard filter" do "event:page==/**\\|page|/other/page" - |> assert_parsed(%{"event:page" => {:matches, "/**\\|page|/other/page"}}) + |> assert_parsed([[:matches, "event:page", "/**\\|page|/other/page"]]) end test "gracefully fails to parse garbage" do "bfg10309\uff1cs1\ufe65s2\u02bas3\u02b9hjl10309" - |> assert_parsed(%{}) + |> assert_parsed([]) end test "gracefully fails to parse garbage with double quotes" do "\";print(md5(31337));$a=\"" - |> assert_parsed(%{}) + |> assert_parsed([]) end test "gracefully fails to parse garbage country code" do "visit:country==AKSJSDFKJSS" - |> assert_parsed(%{}) + |> assert_parsed([]) end test "gracefully fails to parse garbage country code (with pipes)" do "visit:country==ET'||DBMS_PIPE.RECEIVE_MESSAGE(CHR(98)||CHR(98)||CHR(98),15)||'" - |> assert_parsed(%{}) + |> assert_parsed([]) end end end diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs index f568589549..b509b6914c 100644 --- a/test/plausible/stats/query_test.exs +++ b/test/plausible/stats/query_test.exs @@ -194,14 +194,14 @@ defmodule Plausible.Stats.QueryTest do filters = Jason.encode!(%{"goal" => "Signup"}) q = Query.from(site, %{"period" => "6mo", "filters" => filters}) - assert q.filters["event:goal"] == {:is, {:event, "Signup"}} + assert q.filters == [[:is, "event:goal", {:event, "Signup"}]] end test "parses source filter", %{site: site} do filters = Jason.encode!(%{"source" => "Twitter"}) q = Query.from(site, %{"period" => "6mo", "filters" => filters}) - assert q.filters["visit:source"] == {:is, "Twitter"} + assert q.filters == [[:is, "visit:source", "Twitter"]] end end