Refactor internal Query schema and introduce WhereBuilder (#4082)

* New struct format for query after parsing

* WIP refactoring

* WIP: Validations working

* WIP: tuple to list

* continued refactoring

* WIP: parsing defaults

* Breakdown tests pass

* Window functions fix

* Fix default

* Remove dead argument

* Update filters tests

* Update query_test.exs

* Fix table_decider

* sources tests pass

* Filter suggestions fix

* revenue/goal filter applied refactor

* Update top_stats matching

* Get stats_controller tests passing

* Update neighbor_aggregate_time_on_page

* Refactor Query.remove_event_filters into Query.remove_filters, add new callsites

* Move goal where clause building to new WhereBuilder module

* Move event:name filters

* Move more filters to WhereBuilder

* Update fragment to allow non-static meta columns

* Build where clause for events table using WhereBuilder

* Build sessions table where clause using WhereBuilder

* Move time range filtering and site checking to WhereBuilder

* WhereBuilder.build_condition method

* Remove TODO

* _rest pattern for TableDecider, Query pattern matching

Future-proofing in a tiny way

* Hacky fix to get tests passing for Google API tests

* Typespec fix

* Merge conflict

* refactor special goal filter logic in imported.ex

* Docs feedback

* put_filter

---------

Co-authored-by: Robert Joonas <robertjoonas16@gmail.com>

extra/lib/plausible/stats/goal/revenue.ex

@@ -4,6 +4,8 @@ defmodule Plausible.Stats.Goal.Revenue do
   """
   import Ecto.Query
 
+  alias Plausible.Stats.Query
+
   @revenue_metrics [:average_revenue, :total_revenue]
 
   def revenue_metrics() do
@@ -37,10 +39,10 @@ defmodule Plausible.Stats.Goal.Revenue do
   """
   def get_revenue_tracking_currency(site, query, metrics) do
     goal_filters =
-      case query.filters do
-        %{"event:goal" => {:is, {_, goal_name}}} -> [goal_name]
-        %{"event:goal" => {:member, list}} -> Enum.map(list, fn {_, goal_name} -> goal_name end)
-        _any -> []
+      case Query.get_filter(query, "event:goal") do
+        [:is, "event:goal", {_, goal_name}] -> [goal_name]
+        [:member, "event:goal", list] -> Enum.map(list, fn {_, goal_name} -> goal_name end)
+        _ -> []
       end
 
     requested_revenue_metrics? = Enum.any?(metrics, &(&1 in @revenue_metrics))

lib/plausible/google/api.ex

@@ -74,16 +74,16 @@ defmodule Plausible.Google.API do
     end
   end
 
-  def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do
+  def fetch_stats(site, query, limit) do
     with {:ok, site} <- ensure_search_console_property(site),
          {:ok, access_token} <- maybe_refresh_token(site.google_auth),
          {:ok, search_console_filters} <-
-           SearchConsole.Filters.transform(site.google_auth.property, filters),
+           SearchConsole.Filters.transform(site.google_auth.property, query.filters),
          {:ok, stats} <-
            HTTP.list_stats(
              access_token,
              site.google_auth.property,
-             date_range,
+             query.date_range,
              limit,
              search_console_filters
            ) do

lib/plausible/google/search_console/filters.ex

@@ -3,12 +3,11 @@ defmodule Plausible.Google.SearchConsole.Filters do
   import Plausible.Stats.Base, only: [page_regex: 1]
 
   def transform(property, plausible_filters) do
-    plausible_filters = Map.drop(plausible_filters, ["visit:source"])
-
     search_console_filters =
       Enum.reduce_while(plausible_filters, [], fn plausible_filter, search_console_filters ->
         case transform_filter(property, plausible_filter) do
           :unsupported -> {:halt, :unsupported_filters}
+          :ignore -> {:cont, search_console_filters}
           search_console_filter -> {:cont, [search_console_filter | search_console_filters]}
         end
       end)
@@ -20,27 +19,27 @@ defmodule Plausible.Google.SearchConsole.Filters do
     end
   end
 
-  defp transform_filter(property, {"event:page", filter}) do
-    transform_filter(property, {"visit:entry_page", filter})
+  defp transform_filter(property, [op, "event:page" | rest]) do
+    transform_filter(property, [op, "visit:entry_page" | rest])
   end
 
-  defp transform_filter(property, {"visit:entry_page", {:is, page}}) when is_binary(page) do
+  defp transform_filter(property, [:is, "visit:entry_page", page]) when is_binary(page) do
     %{dimension: "page", expression: property_url(property, page)}
   end
 
-  defp transform_filter(property, {"visit:entry_page", {:member, pages}}) when is_list(pages) do
+  defp transform_filter(property, [:member, "visit:entry_page", pages]) when is_list(pages) do
     expression =
       Enum.map_join(pages, "|", fn page -> property_url(property, Regex.escape(page)) end)
 
     %{dimension: "page", operator: "includingRegex", expression: expression}
   end
 
-  defp transform_filter(property, {"visit:entry_page", {:matches, page}}) when is_binary(page) do
+  defp transform_filter(property, [:matches, "visit:entry_page", page]) when is_binary(page) do
     page = page_regex(property_url(property, page))
     %{dimension: "page", operator: "includingRegex", expression: page}
   end
 
-  defp transform_filter(property, {"visit:entry_page", {:matches_member, pages}})
+  defp transform_filter(property, [:matches_member, "visit:entry_page", pages])
        when is_list(pages) do
     expression =
       Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end)
@@ -48,25 +47,27 @@ defmodule Plausible.Google.SearchConsole.Filters do
     %{dimension: "page", operator: "includingRegex", expression: expression}
   end
 
-  defp transform_filter(_property, {"visit:screen", {:is, device}}) when is_binary(device) do
+  defp transform_filter(_property, [:is, "visit:screen", device]) when is_binary(device) do
     %{dimension: "device", expression: search_console_device(device)}
   end
 
-  defp transform_filter(_property, {"visit:screen", {:member, devices}}) when is_list(devices) do
+  defp transform_filter(_property, [:member, "visit:screen", devices]) when is_list(devices) do
     expression = devices |> Enum.join("|")
     %{dimension: "device", operator: "includingRegex", expression: expression}
   end
 
-  defp transform_filter(_property, {"visit:country", {:is, country}}) when is_binary(country) do
+  defp transform_filter(_property, [:is, "visit:country", country]) when is_binary(country) do
     %{dimension: "country", expression: search_console_country(country)}
   end
 
-  defp transform_filter(_property, {"visit:country", {:member, countries}})
+  defp transform_filter(_property, [:member, "visit:country", countries])
        when is_list(countries) do
     expression = Enum.map_join(countries, "|", &search_console_country/1)
     %{dimension: "country", operator: "includingRegex", expression: expression}
   end
 
+  defp transform_filter(_, [_, "visit:source" | _rest]), do: :ignore
+
   defp transform_filter(_, _filter), do: :unsupported
 
   defp property_url("sc-domain:" <> domain, page), do: "https://" <> domain <> page

lib/plausible/stats/aggregate.ex

@@ -70,10 +70,7 @@ defmodule Plausible.Stats.Aggregate do
   defp neighbor_aggregate_time_on_page(site, query) do
     q =
       from(
-        e in base_event_query(site, %Query{
-          query
-          | filters: Map.delete(query.filters, "event:page")
-        }),
+        e in base_event_query(site, Query.remove_filters(query, ["event:page"])),
         select: {
           fragment("? as p", e.pathname),
           fragment("? as t", e.timestamp),
@@ -86,32 +83,32 @@ defmodule Plausible.Stats.Aggregate do
     where_param_idx = length(base_query_raw_params)
 
     {where_clause, where_arg} =
-      case query.filters["event:page"] do
-        {:is, page} ->
+      case Query.get_filter(query, "event:page") do
+        [:is, _, page] ->
           {"p = {$#{where_param_idx}:String}", page}
 
-        {:is_not, page} ->
+        [:is_not, _, page] ->
           {"p != {$#{where_param_idx}:String}", page}
 
-        {:member, page} ->
+        [:member, _, page] ->
           {"p IN {$#{where_param_idx}:Array(String)}", page}
 
-        {:not_member, page} ->
+        [:not_member, _, page] ->
           {"p NOT IN {$#{where_param_idx}:Array(String)}", page}
 
-        {:matches, expr} ->
+        [:matches, _, expr] ->
           regex = page_regex(expr)
           {"match(p, {$#{where_param_idx}:String})", regex}
 
-        {:matches_member, exprs} ->
+        [:matches_member, _, exprs] ->
           page_regexes = Enum.map(exprs, &page_regex/1)
           {"multiMatchAny(p, {$#{where_param_idx}:Array(String)})", page_regexes}
 
-        {:not_matches_member, exprs} ->
+        [:not_matches_member, _, exprs] ->
           page_regexes = Enum.map(exprs, &page_regex/1)
           {"not(multiMatchAny(p, {$#{where_param_idx}:Array(String)}))", page_regexes}
 
-        {:does_not_match, expr} ->
+        [:does_not_match, _, expr] ->
           regex = page_regex(expr)
           {"not(match(p, {$#{where_param_idx}:String}))", regex}
       end
@@ -148,29 +145,29 @@ defmodule Plausible.Stats.Aggregate do
 
   defp window_aggregate_time_on_page(site, query) do
     windowed_pages_q =
-      from e in base_event_query(site, %Query{
-             query
-             | filters: Map.delete(query.filters, "event:page")
-           }),
-           select: %{
-             next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
-             next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
-             timestamp: e.timestamp,
-             pathname: e.pathname,
-             session_id: e.session_id
-           },
-           windows: [
-             event_horizon: [
-               partition_by: e.session_id,
-               order_by: e.timestamp,
-               frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
-             ]
-           ]
+      from e in base_event_query(site, Query.remove_filters(query, ["event:page"])),
+        select: %{
+          next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
+          next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
+          timestamp: e.timestamp,
+          pathname: e.pathname,
+          session_id: e.session_id
+        },
+        windows: [
+          event_horizon: [
+            partition_by: e.session_id,
+            order_by: e.timestamp,
+            frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
+          ]
+        ]
+
+    event_page_filter = Query.get_filter(query, "event:page")
 
     timed_page_transitions_q =
       from e in Ecto.Query.subquery(windowed_pages_q),
         group_by: [e.pathname, e.next_pathname, e.session_id],
-        where: ^Plausible.Stats.Base.dynamic_filter_condition(query, "event:page", :pathname),
+        where:
+          ^Plausible.Stats.Filters.WhereBuilder.build_condition(:pathname, event_page_filter),
         where: e.next_timestamp != 0,
         select: %{
           pathname: e.pathname,

lib/plausible/stats/base.ex

@@ -7,9 +7,6 @@ defmodule Plausible.Stats.Base do
   alias Plausible.Timezones
   import Ecto.Query
 
-  @no_ref "Direct / None"
-  @not_set "(not set)"
-
   @uniq_users_expression "toUInt64(round(uniq(?) * any(_sample_factor)))"
 
   def base_event_query(site, query) do
@@ -30,180 +27,30 @@ defmodule Plausible.Stats.Base do
         on: e.session_id == sq.session_id
       )
     else
-      if query.experimental_reduced_joins? do
-        events_q |> filter_by_visit_props(Filters.event_table_visit_props(), query)
-      else
-        events_q
-      end
+      events_q
     end
   end
 
   defp query_events(site, query) do
-    {first_datetime, last_datetime} = utc_boundaries(query, site)
-
-    q =
-      from(
-        e in "events_v2",
-        where: e.site_id == ^site.id,
-        where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
-      )
+    q = from(e in "events_v2", where: ^Filters.WhereBuilder.build(:events, site, query))
 
     on_ee do
       q = Plausible.Stats.Sampling.add_query_hint(q, query)
     end
 
-    q =
-      q
-      |> where([e], ^dynamic_filter_condition(query, "event:page", :pathname))
-      |> where([e], ^dynamic_filter_condition(query, "event:hostname", :hostname))
-
-    q =
-      case query.filters["event:name"] do
-        {:is, name} ->
-          from(e in q, where: e.name == ^name)
-
-        {:member, list} ->
-          from(e in q, where: e.name in ^list)
-
-        nil ->
-          q
-      end
-
-    q =
-      case query.filters["event:goal"] do
-        {:is, {:page, path}} ->
-          from(e in q, where: e.pathname == ^path and e.name == "pageview")
-
-        {:matches, {:page, expr}} ->
-          regex = page_regex(expr)
-
-          from(e in q,
-            where: fragment("match(?, ?)", e.pathname, ^regex) and e.name == "pageview"
-          )
-
-        {:is, {:event, event}} ->
-          from(e in q, where: e.name == ^event)
-
-        {:member, clauses} ->
-          {events, pages} = split_goals(clauses)
-
-          from(e in q,
-            where: (e.pathname in ^pages and e.name == "pageview") or e.name in ^events
-          )
-
-        {:matches_member, clauses} ->
-          {events, pages} = split_goals(clauses, &page_regex/1)
-
-          event_clause =
-            if Enum.any?(events) do
-              dynamic([x], fragment("multiMatchAny(?, ?)", x.name, ^events))
-            else
-              dynamic([x], false)
-            end
-
-          page_clause =
-            if Enum.any?(pages) do
-              dynamic(
-                [x],
-                fragment("multiMatchAny(?, ?)", x.pathname, ^pages) and x.name == "pageview"
-              )
-            else
-              dynamic([x], false)
-            end
-
-          where_clause = dynamic([], ^event_clause or ^page_clause)
-
-          from(e in q, where: ^where_clause)
-
-        nil ->
-          q
-      end
-
-    q =
-      Enum.reduce(
-        Query.get_all_filters_by_prefix(query, "event:props"),
-        q,
-        &filter_by_custom_prop/2
-      )
-
     q
   end
 
   def query_sessions(site, query) do
-    {first_datetime, last_datetime} =
-      utc_boundaries(query, site)
-
-    q = from(s in "sessions_v2", where: s.site_id == ^site.id)
-
-    sessions_q =
-      if query.experimental_session_count? do
-        from s in q, where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
-      else
-        from s in q, where: s.start >= ^first_datetime and s.start < ^last_datetime
-      end
+    q = from(s in "sessions_v2", where: ^Filters.WhereBuilder.build(:sessions, site, query))
 
     on_ee do
-      sessions_q = Plausible.Stats.Sampling.add_query_hint(sessions_q, query)
+      q = Plausible.Stats.Sampling.add_query_hint(q, query)
     end
 
-    filter_by_entry_props(sessions_q, query)
-    |> filter_by_visit_props(Filters.visit_props(), query)
+    q
   end
 
-  defp filter_by_visit_props(q, visit_props, query) do
-    Enum.reduce(visit_props, q, fn prop_name, sessions_q ->
-      filter_key = "visit:" <> prop_name
-      db_field = String.to_existing_atom(prop_name)
-
-      from(s in sessions_q,
-        where: ^dynamic_filter_condition(query, filter_key, db_field)
-      )
-    end)
-  end
-
-  def filter_by_entry_props(sessions_q, query) do
-    case Query.get_filter_by_prefix(query, "visit:entry_props:") do
-      nil ->
-        sessions_q
-
-      {"visit:entry_props:" <> prop_name, filter_value} ->
-        apply_entry_prop_filter(sessions_q, prop_name, filter_value)
-    end
-  end
-
-  def apply_entry_prop_filter(sessions_q, prop_name, {:is, "(none)"}) do
-    from(
-      s in sessions_q,
-      where: not has_key(s, :entry_meta, ^prop_name)
-    )
-  end
-
-  def apply_entry_prop_filter(sessions_q, prop_name, {:is, value}) do
-    from(
-      s in sessions_q,
-      where:
-        has_key(s, :entry_meta, ^prop_name) and get_by_key(s, :entry_meta, ^prop_name) == ^value
-    )
-  end
-
-  def apply_entry_prop_filter(sessions_q, prop_name, {:is_not, "(none)"}) do
-    from(
-      s in sessions_q,
-      where: has_key(s, :entry_meta, ^prop_name)
-    )
-  end
-
-  def apply_entry_prop_filter(sessions_q, prop_name, {:is_not, value}) do
-    from(
-      s in sessions_q,
-      where:
-        not has_key(s, :entry_meta, ^prop_name) or
-          get_by_key(s, :entry_meta, ^prop_name) != ^value
-    )
-  end
-
-  def apply_entry_prop_filter(sessions_q, _, _), do: sessions_q
-
   def select_event_metrics(metrics) do
     metrics
     |> Enum.map(&select_event_metric/1)
@@ -272,7 +119,9 @@ defmodule Plausible.Stats.Base do
   end
 
   defp select_session_metric(:bounce_rate, query) do
-    condition = dynamic_filter_condition(query, "event:page", :entry_page)
+    # :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page.
+    event_page_filter = Query.get_filter(query, "event:page")
+    condition = Filters.WhereBuilder.build_condition(:entry_page, event_page_filter)
 
     %{
       bounce_rate:
@@ -358,45 +207,6 @@ defmodule Plausible.Stats.Base do
 
   defp select_session_metric(:percentage, _query), do: %{}
 
-  def dynamic_filter_condition(query, filter_key, db_field) do
-    case query && query.filters && query.filters[filter_key] do
-      {:is, value} ->
-        value = db_field_val(db_field, value)
-        dynamic([x], field(x, ^db_field) == ^value)
-
-      {:is_not, value} ->
-        value = db_field_val(db_field, value)
-        dynamic([x], field(x, ^db_field) != ^value)
-
-      {:matches_member, glob_exprs} ->
-        page_regexes = Enum.map(glob_exprs, &page_regex/1)
-        dynamic([x], fragment("multiMatchAny(?, ?)", field(x, ^db_field), ^page_regexes))
-
-      {:not_matches_member, glob_exprs} ->
-        page_regexes = Enum.map(glob_exprs, &page_regex/1)
-        dynamic([x], fragment("not(multiMatchAny(?, ?))", field(x, ^db_field), ^page_regexes))
-
-      {:matches, glob_expr} ->
-        regex = page_regex(glob_expr)
-        dynamic([x], fragment("match(?, ?)", field(x, ^db_field), ^regex))
-
-      {:does_not_match, glob_expr} ->
-        regex = page_regex(glob_expr)
-        dynamic([x], fragment("not(match(?, ?))", field(x, ^db_field), ^regex))
-
-      {:member, list} ->
-        list = Enum.map(list, &db_field_val(db_field, &1))
-        dynamic([x], field(x, ^db_field) in ^list)
-
-      {:not_member, list} ->
-        list = Enum.map(list, &db_field_val(db_field, &1))
-        dynamic([x], field(x, ^db_field) not in ^list)
-
-      nil ->
-        true
-    end
-  end
-
   def filter_converted_sessions(db_query, site, query) do
     if Query.has_event_filters?(query) do
       converted_sessions =
@@ -416,16 +226,6 @@ defmodule Plausible.Stats.Base do
     end
   end
 
-  defp db_field_val(:source, @no_ref), do: ""
-  defp db_field_val(:referrer, @no_ref), do: ""
-  defp db_field_val(:utm_medium, @no_ref), do: ""
-  defp db_field_val(:utm_source, @no_ref), do: ""
-  defp db_field_val(:utm_campaign, @no_ref), do: ""
-  defp db_field_val(:utm_content, @no_ref), do: ""
-  defp db_field_val(:utm_term, @no_ref), do: ""
-  defp db_field_val(_, @not_set), do: ""
-  defp db_field_val(_, val), do: val
-
   defp beginning_of_time(candidate, native_stats_start_at) do
     if Timex.after?(native_stats_start_at, candidate) do
       native_stats_start_at
@@ -486,92 +286,6 @@ defmodule Plausible.Stats.Base do
     "^#{escaped}$"
   end
 
-  defp split_goals(clauses, map_fn \\ &Function.identity/1) do
-    groups =
-      Enum.group_by(clauses, fn {goal_type, _v} -> goal_type end, fn {_k, val} -> map_fn.(val) end)
-
-    {
-      Map.get(groups, :event, []),
-      Map.get(groups, :page, [])
-    }
-  end
-
-  defp filter_by_custom_prop({"event:props:" <> prop_name, {:is, "(none)"}}, q) do
-    from(
-      e in q,
-      where: not has_key(e, :meta, ^prop_name)
-    )
-  end
-
-  defp filter_by_custom_prop({"event:props:" <> prop_name, {:is, value}}, q) do
-    from(
-      e in q,
-      where: has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) == ^value
-    )
-  end
-
-  defp filter_by_custom_prop({"event:props:" <> prop_name, {:is_not, "(none)"}}, q) do
-    from(
-      e in q,
-      where: has_key(e, :meta, ^prop_name)
-    )
-  end
-
-  defp filter_by_custom_prop({"event:props:" <> prop_name, {:is_not, value}}, q) do
-    from(
-      e in q,
-      where: not has_key(e, :meta, ^prop_name) or get_by_key(e, :meta, ^prop_name) != ^value
-    )
-  end
-
-  defp filter_by_custom_prop({"event:props:" <> prop_name, {:matches, value}}, q) do
-    regex = page_regex(value)
-
-    from(
-      e in q,
-      where:
-        has_key(e, :meta, ^prop_name) and
-          fragment("match(?, ?)", get_by_key(e, :meta, ^prop_name), ^regex)
-    )
-  end
-
-  defp filter_by_custom_prop({"event:props:" <> prop_name, {:member, values}}, q) do
-    none_value_included = Enum.member?(values, "(none)")
-
-    from(
-      e in q,
-      where:
-        (has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) in ^values) or
-          (^none_value_included and not has_key(e, :meta, ^prop_name))
-    )
-  end
-
-  defp filter_by_custom_prop({"event:props:" <> prop_name, {:not_member, values}}, q) do
-    none_value_included = Enum.member?(values, "(none)")
-
-    from(
-      e in q,
-      where:
-        (has_key(e, :meta, ^prop_name) and
-           get_by_key(e, :meta, ^prop_name) not in ^values) or
-          (^none_value_included and
-             has_key(e, :meta, ^prop_name) and
-             get_by_key(e, :meta, ^prop_name) not in ^values) or
-          (not (^none_value_included) and not has_key(e, :meta, ^prop_name))
-    )
-  end
-
-  defp filter_by_custom_prop({"event:props:" <> prop_name, {:matches_member, clauses}}, q) do
-    regexes = Enum.map(clauses, &page_regex/1)
-
-    from(
-      e in q,
-      where:
-        has_key(e, :meta, ^prop_name) and
-          fragment("arrayExists(k -> match(?, k), ?)", get_by_key(e, :meta, ^prop_name), ^regexes)
-    )
-  end
-
   defp total_visitors(site, query) do
     base_event_query(site, query)
     |> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id))
@@ -615,7 +329,7 @@ defmodule Plausible.Stats.Base do
   # filters.
   def maybe_add_conversion_rate(q, site, query, metrics) do
     if :conversion_rate in metrics do
-      total_query = query |> Query.remove_event_filters([:goal, :props])
+      total_query = query |> Query.remove_filters(["event:goal", "event:props"])
 
       # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
       subquery(q)

lib/plausible/stats/breakdown.ex

@@ -39,7 +39,7 @@ defmodule Plausible.Stats.Breakdown do
 
     event_query = %Query{
       query
-      | filters: Map.put(query.filters, "event:name", {:member, events}),
+      | filters: query.filters ++ [[:member, "event:name", events]],
         property: "event:name"
     }
 
@@ -180,7 +180,7 @@ defmodule Plausible.Stats.Breakdown do
 
         pages ->
           query
-          |> Query.put_filter("visit:entry_page", {:member, Enum.map(pages, & &1[:page])})
+          |> Query.put_filter([:member, "visit:entry_page", Enum.map(pages, & &1[:page])])
           |> struct!(property: "visit:entry_page")
       end
 
@@ -256,12 +256,12 @@ defmodule Plausible.Stats.Breakdown do
   end
 
   defp update_hostname(query, visit_prop) do
-    case query.filters["event:hostname"] do
+    case Query.get_filter(query, "event:hostname") do
       nil ->
         query
 
-      some ->
-        Plausible.Stats.Query.put_filter(query, visit_prop, some)
+      [op, "event:hostname", value] ->
+        Plausible.Stats.Query.put_filter(query, [op, visit_prop, value])
     end
   end
 
@@ -353,7 +353,7 @@ defmodule Plausible.Stats.Breakdown do
     import Ecto.Query
 
     windowed_pages_q =
-      from e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])),
+      from e in base_event_query(site, Query.remove_filters(query, ["event:page", "event:props"])),
         select: %{
           next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
           next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
@@ -719,7 +719,8 @@ defmodule Plausible.Stats.Breakdown do
          metrics
        ) do
     if :conversion_rate in metrics do
-      breakdown_total_visitors_query = query |> Query.remove_event_filters([:goal, :props])
+      breakdown_total_visitors_query =
+        query |> Query.remove_filters(["event:goal", "event:props"])
 
       breakdown_total_visitors_q =
         breakdown_fn.(site, breakdown_total_visitors_query, [:visitors])

lib/plausible/stats/custom_props.ex

@@ -9,7 +9,7 @@ defmodule Plausible.Stats.CustomProps do
 
   def fetch_prop_names(site, query) do
     case Query.get_filter_by_prefix(query, "event:props:") do
-      {"event:props:" <> key, _} ->
+      [_op, "event:props:" <> key | _rest] ->
         [key]
 
       _ ->

lib/plausible/stats/email_report.ex

@@ -48,7 +48,7 @@ defmodule Plausible.Stats.EmailReport do
   defp put_top_5_sources(stats, site, query) do
     query =
       query
-      |> Query.put_filter("visit:source", {:is_not, "Direct / None"})
+      |> Query.put_filter([:is_not, "visit:source", "Direct / None"])
       |> struct!(property: "visit:source")
 
     sources = Stats.breakdown(site, query, [:visitors], {5, 1})

lib/plausible/stats/filter_suggestions.ex

@@ -152,10 +152,10 @@ defmodule Plausible.Stats.FilterSuggestions do
   def filter_suggestions(site, query, "prop_value", filter_search) do
     filter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%"
 
-    {"event:props:" <> key, _filter} = Query.get_filter_by_prefix(query, "event:props")
+    [_op, "event:props:" <> key | _rest] = Query.get_filter_by_prefix(query, "event:props")
 
     none_q =
-      from(e in base_event_query(site, Query.remove_event_filters(query, [:props])),
+      from(e in base_event_query(site, Query.remove_filters(query, ["event:props"])),
         select: "(none)",
         where: not has_key(e, :meta, ^key),
         limit: 1

lib/plausible/stats/filters/dashboard_filter_parser.ex

@@ -9,52 +9,76 @@ defmodule Plausible.Stats.Filters.DashboardFilterParser do
   the internal React dashboard API
   """
   def parse_and_prefix(filters_map) do
-    Enum.reduce(filters_map, %{}, fn {name, val}, new_filters ->
+    Enum.flat_map(filters_map, fn {name, val} ->
       cond do
         name in Filters.visit_props() ->
-          Map.put(new_filters, "visit:" <> name, filter_value(name, val))
+          [filter_value("visit:" <> name, val)]
 
         name in Filters.event_props() ->
-          Map.put(new_filters, "event:" <> name, filter_value(name, val))
+          [filter_value("event:" <> name, val)]
 
         name == "props" ->
-          put_parsed_props(new_filters, name, val)
+          parse_props(val)
 
         true ->
-          new_filters
+          []
       end
     end)
   end
 
-  @spec filter_value(String.t(), String.t()) :: {atom(), String.t() | [String.t()]}
   def filter_value(key, val) do
     {is_negated, val} = parse_negated_prefix(val)
     {is_contains, val} = parse_contains_prefix(val)
     is_list = list_expression?(val)
     is_wildcard = String.contains?(key, ["page", "goal", "hostname"]) && wildcard_expression?(val)
     val = if is_list, do: parse_member_list(val), else: remove_escape_chars(val)
-    val = if key == "goal", do: wrap_goal_value(val), else: val
+    val = if key == "event:goal", do: wrap_goal_value(val), else: val
 
     cond do
-      is_negated && is_wildcard && is_list -> {:not_matches_member, val}
-      is_negated && is_contains && is_list -> {:not_matches_member, Enum.map(val, &"**#{&1}**")}
-      is_wildcard && is_list -> {:matches_member, val}
-      is_negated && is_wildcard -> {:does_not_match, val}
-      is_negated && is_list -> {:not_member, val}
-      is_negated && is_contains -> {:does_not_match, "**" <> val <> "**"}
-      is_contains && is_list -> {:matches_member, Enum.map(val, &"**#{&1}**")}
-      is_wildcard && is_list -> {:matches_member, val}
-      is_negated -> {:is_not, val}
-      is_list -> {:member, val}
-      is_contains -> {:matches, "**" <> val <> "**"}
-      is_wildcard -> {:matches, val}
-      true -> {:is, val}
+      is_negated && is_wildcard && is_list ->
+        [:not_matches_member, key, val]
+
+      is_negated && is_contains && is_list ->
+        [:not_matches_member, key, Enum.map(val, &"**#{&1}**")]
+
+      is_wildcard && is_list ->
+        [:matches_member, key, val]
+
+      is_negated && is_wildcard ->
+        [:does_not_match, key, val]
+
+      is_negated && is_list ->
+        [:not_member, key, val]
+
+      is_negated && is_contains ->
+        [:does_not_match, key, "**" <> val <> "**"]
+
+      is_contains && is_list ->
+        [:matches_member, key, Enum.map(val, &"**#{&1}**")]
+
+      is_wildcard && is_list ->
+        [:matches_member, key, val]
+
+      is_negated ->
+        [:is_not, key, val]
+
+      is_list ->
+        [:member, key, val]
+
+      is_contains ->
+        [:matches, key, "**" <> val <> "**"]
+
+      is_wildcard ->
+        [:matches, key, val]
+
+      true ->
+        [:is, key, val]
     end
   end
 
-  defp put_parsed_props(new_filters, name, val) do
-    Enum.reduce(val, new_filters, fn {prop_key, prop_val}, new_filters ->
-      Map.put(new_filters, "event:props:" <> prop_key, filter_value(name, prop_val))
+  defp parse_props(val) do
+    Enum.map(val, fn {prop_key, prop_val} ->
+      filter_value("event:props:" <> prop_key, prop_val)
     end)
   end
 

lib/plausible/stats/filters/filters.ex

@@ -56,24 +56,24 @@ defmodule Plausible.Stats.Filters do
   ### Examples:
 
       iex> Filters.parse("{\\"page\\":\\"/blog/**\\"}")
-      %{"event:page" => {:matches, "/blog/**"}}
+      [[:matches, "event:page", "/blog/**"]]
 
       iex> Filters.parse("visit:browser!=Chrome")
-      %{"visit:browser" => {:is_not, "Chrome"}}
+      [[:is_not, "visit:browser", "Chrome"]]
 
       iex> Filters.parse(nil)
-      %{}
+      []
   """
   def parse(filters) when is_binary(filters) do
     case Jason.decode(filters) do
       {:ok, filters} when is_map(filters) -> DashboardFilterParser.parse_and_prefix(filters)
-      {:ok, _} -> %{}
+      {:ok, _} -> []
       {:error, err} -> StatsAPIFilterParser.parse_filter_expression(err.data)
     end
   end
 
   def parse(filters) when is_map(filters), do: DashboardFilterParser.parse_and_prefix(filters)
-  def parse(_), do: %{}
+  def parse(_), do: []
 
   def without_prefix(property) do
     property

lib/plausible/stats/filters/stats_api_filter_parser.ex

@@ -12,13 +12,12 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do
 
     Enum.map(filters, &parse_single_filter/1)
     |> Enum.reject(fn parsed -> parsed == :error end)
-    |> Enum.into(%{})
   end
 
   defp parse_single_filter(str) do
     case to_kv(str) do
       ["event:goal" = key, raw_value] ->
-        {key, parse_goal_filter(raw_value)}
+        parse_goal_filter(key, raw_value)
 
       [key, raw_value] ->
         is_negated? = String.contains?(str, "!=")
@@ -28,11 +27,11 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do
         final_value = remove_escape_chars(raw_value)
 
         cond do
-          is_wildcard? && is_negated? -> {key, {:does_not_match, raw_value}}
-          is_wildcard? -> {key, {:matches, raw_value}}
-          is_list? -> {key, {:member, parse_member_list(raw_value)}}
-          is_negated? -> {key, {:is_not, final_value}}
-          true -> {key, {:is, final_value}}
+          is_wildcard? && is_negated? -> [:does_not_match, key, raw_value]
+          is_wildcard? -> [:matches, key, raw_value]
+          is_list? -> [:member, key, parse_member_list(raw_value)]
+          is_negated? -> [:is_not, key, final_value]
+          true -> [:is, key, final_value]
         end
         |> reject_invalid_country_codes()
 
@@ -41,7 +40,7 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do
     end
   end
 
-  defp reject_invalid_country_codes({"visit:country", {_, code_or_codes}} = filter) do
+  defp reject_invalid_country_codes([_op, "visit:country", code_or_codes] = filter) do
     code_or_codes
     |> List.wrap()
     |> Enum.reduce_while(filter, fn
@@ -59,7 +58,7 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do
     |> Enum.map(&String.trim/1)
   end
 
-  defp parse_goal_filter(value) do
+  defp parse_goal_filter(key, value) do
     is_list? = list_expression?(value)
     is_wildcard? = wildcard_expression?(value)
 
@@ -72,10 +71,10 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do
       |> wrap_goal_value()
 
     cond do
-      is_list? && is_wildcard? -> {:matches_member, value}
-      is_list? -> {:member, value}
-      is_wildcard? -> {:matches, value}
-      true -> {:is, value}
+      is_list? && is_wildcard? -> [:matches_member, key, value]
+      is_list? -> [:member, key, value]
+      is_wildcard? -> [:matches, key, value]
+      true -> [:is, key, value]
     end
   end
 end

lib/plausible/stats/filters/where_builder.ex

@@ -0,0 +1,304 @@
+defmodule Plausible.Stats.Filters.WhereBuilder do
+  @moduledoc """
+  A module for building am ecto where clause of a query out of a query.
+  """
+
+  import Ecto.Query
+  import Plausible.Stats.Base, only: [page_regex: 1, utc_boundaries: 2]
+
+  alias Plausible.Stats.Query
+
+  use Plausible.Stats.Fragments
+
+  @sessions_only_visit_fields [
+    :entry_page,
+    :exit_page,
+    :entry_page_hostname,
+    :exit_page_hostname
+  ]
+
+  @doc "Builds WHERE clause for a given Query against sessions or events table"
+  def build(table, site, query) do
+    base_condition = filter_site_time_range(table, site, query)
+
+    query.filters
+    |> Enum.map(&add_filter(table, query, &1))
+    |> Enum.reduce(base_condition, fn condition, acc -> dynamic([], ^acc and ^condition) end)
+  end
+
+  @doc """
+  Builds WHERE clause condition based off of a filter and a custom column name
+  Used for special business logic cases
+
+  Accepts nil as the `filter` parameter, in which case the condition is a no-op (WHERE TRUE).
+  """
+  def build_condition(db_field, filter) do
+    if filter do
+      filter_field(db_field, filter)
+    else
+      true
+    end
+  end
+
+  defp filter_site_time_range(:events, site, query) do
+    {first_datetime, last_datetime} = utc_boundaries(query, site)
+
+    dynamic(
+      [e],
+      e.site_id == ^site.id and e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
+    )
+  end
+
+  defp filter_site_time_range(:sessions, site, %Query{experimental_session_count?: true} = query) do
+    {first_datetime, last_datetime} = utc_boundaries(query, site)
+
+    # Counts each _active_ session in time range even if they started before
+    dynamic(
+      [s],
+      s.site_id == ^site.id and s.timestamp >= ^first_datetime and s.start < ^last_datetime
+    )
+  end
+
+  defp filter_site_time_range(:sessions, site, query) do
+    {first_datetime, last_datetime} = utc_boundaries(query, site)
+
+    dynamic(
+      [s],
+      s.site_id == ^site.id and s.start >= ^first_datetime and s.start < ^last_datetime
+    )
+  end
+
+  defp add_filter(:events, _query, [:is, "event:name", name]) do
+    dynamic([e], e.name == ^name)
+  end
+
+  defp add_filter(:events, _query, [:member, "event:name", list]) do
+    dynamic([e], e.name in ^list)
+  end
+
+  defp add_filter(:events, _query, [:is, "event:goal", {:page, path}]) do
+    dynamic([e], e.pathname == ^path and e.name == "pageview")
+  end
+
+  defp add_filter(:events, _query, [:matches, "event:goal", {:page, expr}]) do
+    regex = page_regex(expr)
+
+    dynamic([e], fragment("match(?, ?)", e.pathname, ^regex) and e.name == "pageview")
+  end
+
+  defp add_filter(:events, _query, [:is, "event:goal", {:event, event}]) do
+    dynamic([e], e.name == ^event)
+  end
+
+  defp add_filter(:events, _query, [:member, "event:goal", clauses]) do
+    {events, pages} = split_goals(clauses)
+
+    dynamic([e], (e.pathname in ^pages and e.name == "pageview") or e.name in ^events)
+  end
+
+  defp add_filter(:events, _query, [:matches_member, "event:goal", clauses]) do
+    {events, pages} = split_goals(clauses, &page_regex/1)
+
+    event_clause =
+      if Enum.any?(events) do
+        dynamic([x], fragment("multiMatchAny(?, ?)", x.name, ^events))
+      else
+        dynamic([x], false)
+      end
+
+    page_clause =
+      if Enum.any?(pages) do
+        dynamic(
+          [x],
+          fragment("multiMatchAny(?, ?)", x.pathname, ^pages) and x.name == "pageview"
+        )
+      else
+        dynamic([x], false)
+      end
+
+    where_clause = dynamic([], ^event_clause or ^page_clause)
+
+    dynamic([e], ^where_clause)
+  end
+
+  defp add_filter(:events, _query, [_, "event:page" | _rest] = filter) do
+    filter_field(:pathname, filter)
+  end
+
+  defp add_filter(:events, _query, [_, "event:hostname" | _rest] = filter) do
+    filter_field(:hostname, filter)
+  end
+
+  defp add_filter(:events, _query, [_, "event:props:" <> prop_name | _rest] = filter) do
+    filter_custom_prop(prop_name, :meta, filter)
+  end
+
+  defp add_filter(:events, _query, [_, "visit:entry_props:" <> _prop_name | _rest]) do
+    true
+  end
+
+  defp add_filter(
+         :events,
+         %Query{experimental_reduced_joins?: true},
+         [_, "visit:" <> key | _rest] = filter
+       ) do
+    # Filter events query if experimental_reduced_joins? is true
+    field_name = String.to_existing_atom(key)
+
+    if Enum.member?(@sessions_only_visit_fields, field_name) do
+      true
+    else
+      filter_field(field_name, filter)
+    end
+  end
+
+  defp add_filter(:events, _query, [_, "visit:" <> _key | _rest]) do
+    true
+  end
+
+  defp add_filter(:sessions, _query, [_, "visit:entry_props:" <> prop_name | _rest] = filter) do
+    filter_custom_prop(prop_name, :entry_meta, filter)
+  end
+
+  defp add_filter(:sessions, _query, [_, "visit:" <> key | _rest] = filter) do
+    filter_field(String.to_existing_atom(key), filter)
+  end
+
+  defp add_filter(:sessions, _query, [_, "event:" <> _ | _rest]) do
+    # Cannot apply sessions filters directly on session query where clause.
+    true
+  end
+
+  defp filter_custom_prop(prop_name, column_name, [:is, _, "(none)"]) do
+    dynamic([t], not has_key(t, column_name, ^prop_name))
+  end
+
+  defp filter_custom_prop(prop_name, column_name, [:is, _, value]) do
+    dynamic(
+      [t],
+      has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) == ^value
+    )
+  end
+
+  defp filter_custom_prop(prop_name, column_name, [:is_not, _, "(none)"]) do
+    dynamic([t], has_key(t, column_name, ^prop_name))
+  end
+
+  defp filter_custom_prop(prop_name, column_name, [:is_not, _, value]) do
+    dynamic(
+      [t],
+      not has_key(t, column_name, ^prop_name) or get_by_key(t, column_name, ^prop_name) != ^value
+    )
+  end
+
+  defp filter_custom_prop(prop_name, column_name, [:matches, _, value]) do
+    regex = page_regex(value)
+
+    dynamic(
+      [t],
+      has_key(t, column_name, ^prop_name) and
+        fragment("match(?, ?)", get_by_key(t, column_name, ^prop_name), ^regex)
+    )
+  end
+
+  defp filter_custom_prop(prop_name, column_name, [:member, _, values]) do
+    none_value_included = Enum.member?(values, "(none)")
+
+    dynamic(
+      [t],
+      (has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) in ^values) or
+        (^none_value_included and not has_key(t, column_name, ^prop_name))
+    )
+  end
+
+  defp filter_custom_prop(prop_name, column_name, [:not_member, _, values]) do
+    none_value_included = Enum.member?(values, "(none)")
+
+    dynamic(
+      [t],
+      (has_key(t, column_name, ^prop_name) and
+         get_by_key(t, column_name, ^prop_name) not in ^values) or
+        (^none_value_included and
+           has_key(t, column_name, ^prop_name) and
+           get_by_key(t, column_name, ^prop_name) not in ^values) or
+        (not (^none_value_included) and not has_key(t, column_name, ^prop_name))
+    )
+  end
+
+  defp filter_custom_prop(prop_name, column_name, [:matches_member, _, clauses]) do
+    regexes = Enum.map(clauses, &page_regex/1)
+
+    dynamic(
+      [t],
+      has_key(t, column_name, ^prop_name) and
+        fragment(
+          "arrayExists(k -> match(?, k), ?)",
+          get_by_key(t, column_name, ^prop_name),
+          ^regexes
+        )
+    )
+  end
+
+  defp filter_field(db_field, [:is, _key, value]) do
+    value = db_field_val(db_field, value)
+    dynamic([x], field(x, ^db_field) == ^value)
+  end
+
+  defp filter_field(db_field, [:is_not, _key, value]) do
+    value = db_field_val(db_field, value)
+    dynamic([x], field(x, ^db_field) != ^value)
+  end
+
+  defp filter_field(db_field, [:matches_member, _key, glob_exprs]) do
+    page_regexes = Enum.map(glob_exprs, &page_regex/1)
+    dynamic([x], fragment("multiMatchAny(?, ?)", field(x, ^db_field), ^page_regexes))
+  end
+
+  defp filter_field(db_field, [:not_matches_member, _key, glob_exprs]) do
+    page_regexes = Enum.map(glob_exprs, &page_regex/1)
+    dynamic([x], fragment("not(multiMatchAny(?, ?))", field(x, ^db_field), ^page_regexes))
+  end
+
+  defp filter_field(db_field, [:matches, _key, glob_expr]) do
+    regex = page_regex(glob_expr)
+    dynamic([x], fragment("match(?, ?)", field(x, ^db_field), ^regex))
+  end
+
+  defp filter_field(db_field, [:does_not_match, _key, glob_expr]) do
+    regex = page_regex(glob_expr)
+    dynamic([x], fragment("not(match(?, ?))", field(x, ^db_field), ^regex))
+  end
+
+  defp filter_field(db_field, [:member, _key, list]) do
+    list = Enum.map(list, &db_field_val(db_field, &1))
+    dynamic([x], field(x, ^db_field) in ^list)
+  end
+
+  defp filter_field(db_field, [:not_member, _key, list]) do
+    list = Enum.map(list, &db_field_val(db_field, &1))
+    dynamic([x], field(x, ^db_field) not in ^list)
+  end
+
+  @no_ref "Direct / None"
+  @not_set "(not set)"
+
+  defp db_field_val(:source, @no_ref), do: ""
+  defp db_field_val(:referrer, @no_ref), do: ""
+  defp db_field_val(:utm_medium, @no_ref), do: ""
+  defp db_field_val(:utm_source, @no_ref), do: ""
+  defp db_field_val(:utm_campaign, @no_ref), do: ""
+  defp db_field_val(:utm_content, @no_ref), do: ""
+  defp db_field_val(:utm_term, @no_ref), do: ""
+  defp db_field_val(_, @not_set), do: ""
+  defp db_field_val(_, val), do: val
+
+  defp split_goals(clauses, map_fn \\ &Function.identity/1) do
+    groups =
+      Enum.group_by(clauses, fn {goal_type, _v} -> goal_type end, fn {_k, val} -> map_fn.(val) end)
+
+    {
+      Map.get(groups, :event, []),
+      Map.get(groups, :page, [])
+    }
+  end
+end

lib/plausible/stats/fragments.ex

@@ -110,7 +110,7 @@ defmodule Plausible.Stats.Fragments do
     quote do
       fragment(
         "has(?, ?)",
-        field(unquote(table), unquote(meta_key_column(meta_column))),
+        field(unquote(table), ^meta_key_column(unquote(meta_column))),
         unquote(key)
       )
     end
@@ -130,18 +130,18 @@ defmodule Plausible.Stats.Fragments do
     quote do
       fragment(
         "?[indexOf(?, ?)]",
-        field(unquote(table), unquote(meta_value_column(meta_column))),
-        field(unquote(table), unquote(meta_key_column(meta_column))),
+        field(unquote(table), ^meta_value_column(unquote(meta_column))),
+        field(unquote(table), ^meta_key_column(unquote(meta_column))),
         unquote(key)
       )
     end
   end
 
-  defp meta_key_column(:meta), do: :"meta.key"
-  defp meta_key_column(:entry_meta), do: :"entry_meta.key"
+  def meta_key_column(:meta), do: :"meta.key"
+  def meta_key_column(:entry_meta), do: :"entry_meta.key"
 
-  defp meta_value_column(:meta), do: :"meta.value"
-  defp meta_value_column(:entry_meta), do: :"entry_meta.value"
+  def meta_value_column(:meta), do: :"meta.value"
+  def meta_value_column(:entry_meta), do: :"entry_meta.value"
 
   defmacro __using__(_) do
     quote do

lib/plausible/stats/imported.ex

@@ -33,27 +33,29 @@ defmodule Plausible.Stats.Imported do
 
   @imported_properties Map.keys(@property_to_table_mappings)
 
+  @goals_with_url ["Outbound Link: Click", "Cloaked Link: Click", "File Download"]
+
+  @doc """
+  Returns a boolean indicating whether the combination of filters and
+  breakdown property is possible to query from the imported tables.
+
+  Usually, when no filters are used, the imported schema supports the
+  query. There is one exception though - breakdown by a custom property.
+  We are currently importing only two custom properties - `url` and `path.
+  Both these properties can only be used with their special goal filter
+  (see `@goals_with_url` and `@goals_with_path`).
+  """
   def schema_supports_query?(query) do
-    filter_count = length(Map.keys(query.filters))
+    filter_count = length(query.filters)
 
     case {filter_count, query.property} do
       {0, "event:props:" <> _} -> false
       {0, _} -> true
-      {1, _} -> supports_single_filter?(query)
+      {1, "event:props:url"} -> has_special_goal_filter?(query, @goals_with_url)
       {_, _} -> false
     end
   end
 
-  defp supports_single_filter?(%Query{
-         filters: %{"event:goal" => {:is, {:event, event}}},
-         property: "event:props:url"
-       })
-       when event in ["Outbound Link: Click", "File Download"] do
-    true
-  end
-
-  defp supports_single_filter?(_query), do: false
-
   def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _),
     do: native_q
 
@@ -118,7 +120,7 @@ defmodule Plausible.Stats.Imported do
         where: i.visitors > 0,
         select: %{}
       )
-      |> maybe_apply_filter(query.filters, property, dim)
+      |> maybe_apply_filter(query, property, dim)
       |> group_imported_by(dim)
       |> select_imported_metrics(metrics)
 
@@ -214,23 +216,35 @@ defmodule Plausible.Stats.Imported do
     )
   end
 
-  defp maybe_apply_filter(
-         q,
-         %{"event:goal" => {:is, {:event, event_name}}},
-         "event:props:url",
-         _dim
-       )
-       when event_name in ["Outbound Link: Click", "File Download"] do
-    where(q, [i], i.name == ^event_name)
+  defp maybe_apply_filter(q, query, "event:props:url", _) do
+    if name = find_special_goal_filter(query, @goals_with_url) do
+      where(q, [i], i.name == ^name)
+    else
+      q
+    end
   end
 
-  defp maybe_apply_filter(q, filters, property, dim) do
-    case filters[property] do
-      {:member, list} -> where(q, [i], field(i, ^dim) in ^list)
+  defp maybe_apply_filter(q, query, property, dim) do
+    case Query.get_filter(query, property) do
+      [:member, _, list] -> where(q, [i], field(i, ^dim) in ^list)
       _ -> q
     end
   end
 
+  defp has_special_goal_filter?(query, event_names) do
+    not is_nil(find_special_goal_filter(query, event_names))
+  end
+
+  defp find_special_goal_filter(query, event_names) do
+    case Query.get_filter(query, "event:goal") do
+      [:is, "event:goal", {:event, name}] ->
+        if name in event_names, do: name, else: nil
+
+      _ ->
+        nil
+    end
+  end
+
   defp select_imported_metrics(q, []), do: q
 
   defp select_imported_metrics(q, [:visitors | rest]) do

lib/plausible/stats/query.ex

@@ -5,7 +5,7 @@ defmodule Plausible.Stats.Query do
             interval: nil,
             period: nil,
             property: nil,
-            filters: %{},
+            filters: [],
             sample_threshold: 20_000_000,
             imported_data_requested: false,
             include_imported: false,
@@ -200,50 +200,36 @@ defmodule Plausible.Stats.Query do
     struct!(query, filters: Filters.parse(params["filters"]))
   end
 
-  def put_filter(query, key, val) do
-    parsed_val =
-      if is_binary(val) do
-        Filters.DashboardFilterParser.filter_value(key, val)
-      else
-        val
-      end
-
+  def put_filter(query, filter) do
     struct!(query,
-      filters: Map.put(query.filters, key, parsed_val)
+      filters: query.filters ++ [filter]
     )
   end
 
-  def remove_event_filters(query, opts) do
+  def remove_filters(query, prefixes) do
     new_filters =
-      Enum.filter(query.filters, fn {filter_key, _} ->
-        cond do
-          :page in opts && filter_key == "event:page" -> false
-          :goal in opts && filter_key == "event:goal" -> false
-          :props in opts && filter_key && String.starts_with?(filter_key, "event:props:") -> false
-          true -> true
-        end
+      Enum.reject(query.filters, fn [_, filter_key | _rest] ->
+        Enum.any?(prefixes, &String.starts_with?(filter_key, &1))
       end)
-      |> Enum.into(%{})
 
     struct!(query, filters: new_filters)
   end
 
   def has_event_filters?(query) do
-    Enum.any?(query.filters, fn
-      {"event:" <> _, _} -> true
-      _ -> false
+    Enum.any?(query.filters, fn [_op, prop | _rest] ->
+      String.starts_with?(prop, "event:")
     end)
   end
 
   def get_filter_by_prefix(query, prefix) do
-    Enum.find(query.filters, fn {prop, _value} ->
+    Enum.find(query.filters, fn [_op, prop | _rest] ->
       String.starts_with?(prop, prefix)
     end)
   end
 
-  def get_all_filters_by_prefix(query, prefix) do
-    Enum.filter(query.filters, fn {prop, _value} ->
-      String.starts_with?(prop, prefix)
+  def get_filter(query, name) do
+    Enum.find(query.filters, fn [_op, prop | _rest] ->
+      prop == name
     end)
   end
 
@@ -281,7 +267,12 @@ defmodule Plausible.Stats.Query do
 
   @spec trace(%__MODULE__{}, [atom()]) :: %__MODULE__{}
   def trace(%__MODULE__{} = query, metrics) do
-    filter_keys = Map.keys(query.filters) |> Enum.sort() |> Enum.join(";")
+    filter_keys =
+      query.filters
+      |> Enum.map(fn [_op, prop | _rest] -> prop end)
+      |> Enum.sort()
+      |> Enum.join(";")
+
     metrics = metrics |> Enum.sort() |> Enum.join(";")
 
     Tracer.set_attributes([

lib/plausible/stats/table_decider.ex

@@ -23,15 +23,15 @@ defmodule Plausible.Stats.TableDecider do
       partition(metrics, query, &metric_partitioner/2)
 
     # Treat breakdown property as yet another filter
-    filters =
+    query =
       if breakdown_property do
-        Map.put(query.filters, breakdown_property, nil)
+        Query.put_filter(query, [:is, breakdown_property, []])
       else
-        query.filters
+        query
       end
 
     %{event: event_only_filters, session: session_only_filters} =
-      partition(filters, query, &filters_partitioner/2)
+      partition(query.filters, query, &filters_partitioner/2)
 
     cond do
       # Only one table needs to be queried
@@ -83,16 +83,16 @@ defmodule Plausible.Stats.TableDecider do
 
   defp metric_partitioner(_, _), do: :either
 
-  defp filters_partitioner(_, {"event:" <> _, _}), do: :event
-  defp filters_partitioner(_, {"visit:entry_page", _}), do: :session
-  defp filters_partitioner(_, {"visit:entry_page_hostname", _}), do: :session
-  defp filters_partitioner(_, {"visit:exit_page", _}), do: :session
-  defp filters_partitioner(_, {"visit:exit_page_hostname", _}), do: :session
+  defp filters_partitioner(_, [_, "event:" <> _ | _rest]), do: :event
+  defp filters_partitioner(_, [_, "visit:entry_page" | _rest]), do: :session
+  defp filters_partitioner(_, [_, "visit:entry_page_hostname" | _rest]), do: :session
+  defp filters_partitioner(_, [_, "visit:exit_page" | _rest]), do: :session
+  defp filters_partitioner(_, [_, "visit:exit_page_hostname" | _rest]), do: :session
 
-  defp filters_partitioner(%Query{experimental_reduced_joins?: true}, {"visit:" <> _, _}),
+  defp filters_partitioner(%Query{experimental_reduced_joins?: true}, [_, "visit:" <> _ | _rest]),
     do: :either
 
-  defp filters_partitioner(_, {"visit:" <> _, _}),
+  defp filters_partitioner(_, [_, "visit:" <> _ | _rest]),
     do: :session
 
   defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) do

lib/plausible/stats/timeseries.ex

@@ -314,7 +314,7 @@ defmodule Plausible.Stats.Timeseries do
 
   defp maybe_add_timeseries_conversion_rate(q, site, query, metrics) do
     if :conversion_rate in metrics do
-      totals_query = query |> Query.remove_event_filters([:goal, :props])
+      totals_query = query |> Query.remove_filters(["event:goal", "event:props"])
 
       totals_timeseries_q =
         from(e in base_event_query(site, totals_query),

lib/plausible_web/controllers/api/external_stats_controller.ex

@@ -119,7 +119,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
         {_, _, :all} ->
           true
 
-        {{"event:props:" <> prop, _}, _property, allowed_props} ->
+        {[_, "event:props:" <> prop | _], _property, allowed_props} ->
           prop in allowed_props
 
         {_filter, "event:props:" <> prop, allowed_props} ->
@@ -156,10 +156,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
 
   defp validate_metric("time_on_page" = metric, query) do
     cond do
-      query.filters["event:goal"] ->
+      Query.get_filter(query, "event:goal") ->
         {:error, "Metric `#{metric}` cannot be queried when filtering by `event:goal`"}
 
-      query.filters["event:name"] ->
+      Query.get_filter(query, "event:name") ->
         {:error, "Metric `#{metric}` cannot be queried when filtering by `event:name`"}
 
       query.property == "event:page" ->
@@ -169,7 +169,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
         {:error,
          "Metric `#{metric}` is not supported in breakdown queries (except `event:page` breakdown)"}
 
-      query.filters["event:page"] ->
+      Query.get_filter(query, "event:page") ->
         {:ok, metric}
 
       true ->
@@ -183,7 +183,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
       query.property == "event:goal" ->
         {:ok, metric}
 
-      query.filters["event:goal"] ->
+      Query.get_filter(query, "event:goal") ->
         {:ok, metric}
 
       true ->
@@ -198,7 +198,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
 
   defp validate_metric("views_per_visit" = metric, query) do
     cond do
-      query.filters["event:page"] ->
+      Query.get_filter(query, "event:page") ->
         {:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."}
 
       query.property != nil ->
@@ -235,7 +235,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
   end
 
   defp find_event_only_filter(query) do
-    Map.keys(query.filters) |> Enum.find(&event_only_property?/1)
+    query.filters
+    |> Enum.map(fn [_op, prop | _] -> prop end)
+    |> Enum.find(&event_only_property?/1)
   end
 
   defp event_only_property?("event:name"), do: true
@@ -327,7 +329,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
     end)
   end
 
-  defp validate_filter(site, {"event:goal", {_type, goal_filter}}) do
+  defp validate_filter(site, [_type, "event:goal", goal_filter]) do
     configured_goals =
       Plausible.Goals.for_site(site)
       |> Enum.map(fn
@@ -350,7 +352,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
     end
   end
 
-  defp validate_filter(_site, {property, _}) do
+  defp validate_filter(_site, [_, property | _]) do
     if Plausible.Stats.Props.valid_prop?(property) do
       :ok
     else

lib/plausible_web/controllers/api/stats_controller.ex

@@ -6,6 +6,7 @@ defmodule PlausibleWeb.Api.StatsController do
 
   alias Plausible.Stats
   alias Plausible.Stats.{Query, Comparisons}
+  alias Plausible.Stats.Filters.DashboardFilterParser
   alias PlausibleWeb.Api.Helpers, as: H
 
   require Logger
@@ -276,11 +277,25 @@ defmodule PlausibleWeb.Api.StatsController do
     end
   end
 
-  defp fetch_top_stats(
-         site,
-         %Query{period: "realtime", filters: %{"event:goal" => _goal}} = query,
-         _comparison_query
-       ) do
+  defp fetch_top_stats(site, query, comparison_query) do
+    goal_filter = Query.get_filter(query, "event:goal")
+
+    cond do
+      query.period == "realtime" && goal_filter ->
+        fetch_goal_realtime_top_stats(site, query, comparison_query)
+
+      query.period == "realtime" ->
+        fetch_realtime_top_stats(site, query, comparison_query)
+
+      goal_filter ->
+        fetch_goal_top_stats(site, query, comparison_query)
+
+      true ->
+        fetch_other_top_stats(site, query, comparison_query)
+    end
+  end
+
+  defp fetch_goal_realtime_top_stats(site, query, _comparison_query) do
     query_30m = %Query{query | period: "30m"}
 
     %{
@@ -308,7 +323,7 @@ defmodule PlausibleWeb.Api.StatsController do
     {stats, 100}
   end
 
-  defp fetch_top_stats(site, %Query{period: "realtime"} = query, _comparison_query) do
+  defp fetch_realtime_top_stats(site, query, _comparison_query) do
     query_30m = %Query{query | period: "30m"}
 
     %{
@@ -336,7 +351,7 @@ defmodule PlausibleWeb.Api.StatsController do
     {stats, 100}
   end
 
-  defp fetch_top_stats(site, %Query{filters: %{"event:goal" => _}} = query, comparison_query) do
+  defp fetch_goal_top_stats(site, query, comparison_query) do
     metrics =
       [:total_visitors, :visitors, :events, :conversion_rate] ++ @revenue_metrics
 
@@ -365,9 +380,9 @@ defmodule PlausibleWeb.Api.StatsController do
     |> then(&{&1, 100})
   end
 
-  defp fetch_top_stats(site, query, comparison_query) do
+  defp fetch_other_top_stats(site, query, comparison_query) do
     metrics =
-      if query.filters["event:page"] do
+      if Query.get_filter(query, "event:page") do
         [
           :visitors,
           :visits,
@@ -469,7 +484,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{source: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         res
         |> transform_keys(%{visitors: :conversions})
         |> to_csv([:name, :conversions, :conversion_rate])
@@ -519,17 +534,17 @@ defmodule PlausibleWeb.Api.StatsController do
     end
 
     defp validate_funnel_query(query) do
-      case query do
-        _ when is_map_key(query.filters, "event:goal") ->
+      cond do
+        Query.get_filter(query, "event:goal") ->
           {:error, {:invalid_funnel_query, "goals"}}
 
-        _ when is_map_key(query.filters, "event:page") ->
+        Query.get_filter(query, "event:page") ->
           {:error, {:invalid_funnel_query, "pages"}}
 
-        _ when query.period == "realtime" ->
+        query.period == "realtime" ->
           {:error, {:invalid_funnel_query, "realtime period"}}
 
-        _ ->
+        true ->
           :ok
       end
     end
@@ -547,7 +562,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{utm_medium: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         res
         |> transform_keys(%{visitors: :conversions})
         |> to_csv([:name, :conversions, :conversion_rate])
@@ -571,7 +586,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{utm_campaign: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         res
         |> transform_keys(%{visitors: :conversions})
         |> to_csv([:name, :conversions, :conversion_rate])
@@ -595,7 +610,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{utm_content: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         res
         |> transform_keys(%{visitors: :conversions})
         |> to_csv([:name, :conversions, :conversion_rate])
@@ -619,7 +634,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{utm_term: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         res
         |> transform_keys(%{visitors: :conversions})
         |> to_csv([:name, :conversions, :conversion_rate])
@@ -643,7 +658,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{utm_source: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         res
         |> transform_keys(%{visitors: :conversions})
         |> to_csv([:name, :conversions, :conversion_rate])
@@ -667,7 +682,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{referrer: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         res
         |> transform_keys(%{visitors: :conversions})
         |> to_csv([:name, :conversions, :conversion_rate])
@@ -711,9 +726,11 @@ defmodule PlausibleWeb.Api.StatsController do
     site = conn.assigns[:site]
     params = Map.put(params, "property", "visit:referrer")
 
+    referrer_filter = DashboardFilterParser.filter_value("visit:source", referrer)
+
     query =
       Query.from(site, params)
-      |> Query.put_filter("visit:source", referrer)
+      |> Query.put_filter(referrer_filter)
 
     pagination = parse_pagination(params)
 
@@ -747,7 +764,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{page: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         pages
         |> transform_keys(%{visitors: :conversions})
         |> to_csv([:name, :conversions, :conversion_rate])
@@ -771,7 +788,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{entry_page: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         to_csv(entry_pages, [:name, :visitors, :conversion_rate], [
           :name,
           :conversions,
@@ -803,7 +820,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{exit_page: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         to_csv(exit_pages, [:name, :visitors, :conversion_rate], [
           :name,
           :conversions,
@@ -829,8 +846,9 @@ defmodule PlausibleWeb.Api.StatsController do
       pages = Enum.map(breakdown_results, & &1[:exit_page])
 
       total_visits_query =
-        Query.put_filter(query, "event:page", {:member, pages})
-        |> Query.put_filter("event:name", {:is, "pageview"})
+        query
+        |> Query.put_filter([:member, "event:page", pages])
+        |> Query.put_filter([:is, "event:name", "pageview"])
         |> struct!(property: "event:page")
 
       total_pageviews =
@@ -870,7 +888,7 @@ defmodule PlausibleWeb.Api.StatsController do
           Map.put(country, :name, country_info.name)
         end)
 
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         countries
         |> transform_keys(%{visitors: :conversions})
         |> to_csv([:name, :conversions, :conversion_rate])
@@ -926,7 +944,7 @@ defmodule PlausibleWeb.Api.StatsController do
       end)
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         regions
         |> transform_keys(%{visitors: :conversions})
         |> to_csv([:name, :conversions, :conversion_rate])
@@ -966,7 +984,7 @@ defmodule PlausibleWeb.Api.StatsController do
       end)
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         cities
         |> transform_keys(%{visitors: :conversions})
         |> to_csv([:name, :conversions, :conversion_rate])
@@ -990,7 +1008,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{browser: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         browsers
         |> transform_keys(%{visitors: :conversions})
         |> to_csv([:name, :conversions, :conversion_rate])
@@ -1014,7 +1032,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{browser_version: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         versions
         |> transform_keys(%{
           name: :version,
@@ -1044,7 +1062,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{os: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         systems
         |> transform_keys(%{visitors: :conversions})
         |> to_csv([:name, :conversions, :conversion_rate])
@@ -1068,7 +1086,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{os_version: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         versions
         |> transform_keys(%{name: :version, os: :name, visitors: :conversions})
         |> to_csv([:name, :version, :conversions, :conversion_rate])
@@ -1094,7 +1112,7 @@ defmodule PlausibleWeb.Api.StatsController do
       |> transform_keys(%{device: :name})
 
     if params["csv"] do
-      if Map.has_key?(query.filters, "event:goal") do
+      if Query.get_filter(query, "event:goal") do
         sizes
         |> transform_keys(%{visitors: :conversions})
         |> to_csv([:name, :conversions, :conversion_rate])
@@ -1183,7 +1201,7 @@ defmodule PlausibleWeb.Api.StatsController do
         |> Enum.concat()
 
       percent_or_cr =
-        if query.filters["event:goal"],
+        if Query.get_filter(query, "event:goal"),
           do: :conversion_rate,
           else: :percentage
 
@@ -1200,7 +1218,7 @@ defmodule PlausibleWeb.Api.StatsController do
     query = Query.from(site, params)
 
     metrics =
-      if query.filters["event:goal"] do
+      if Query.get_filter(query, "event:goal") do
         [:visitors, :events, :conversion_rate] ++ @revenue_metrics
       else
         [:visitors, :events, :percentage] ++ @revenue_metrics
@@ -1359,7 +1377,7 @@ defmodule PlausibleWeb.Api.StatsController do
 
     requires_goal_filter? = metric in [:conversion_rate, :events]
 
-    if requires_goal_filter? and !query.filters["event:goal"] do
+    if requires_goal_filter? and !Query.get_filter(query, "event:goal") do
       {:error, "Metric `#{metric}` can only be queried with a goal filter"}
     else
       {:ok, metric}
@@ -1401,7 +1419,7 @@ defmodule PlausibleWeb.Api.StatsController do
   end
 
   defp breakdown_metrics(query, extra_metrics \\ []) do
-    if query.filters["event:goal"] do
+    if Query.get_filter(query, "event:goal") do
       [:visitors, :conversion_rate, :total_visitors]
     else
       [:visitors] ++ extra_metrics

lib/plausible_web/controllers/stats_controller.ex

@@ -180,16 +180,28 @@ defmodule PlausibleWeb.StatsController do
     |> Enum.join()
   end
 
-  defp csv_graph_metrics(%Query{filters: %{"event:goal" => _}}) do
-    metrics = [:visitors, :events, :conversion_rate]
-    column_headers = [:date, :unique_conversions, :total_conversions, :conversion_rate]
+  defp csv_graph_metrics(query) do
+    {metrics, column_headers} =
+      if Query.get_filter(query, "event:goal") do
+        {
+          [:visitors, :events, :conversion_rate],
+          [:date, :unique_conversions, :total_conversions, :conversion_rate]
+        }
+      else
+        metrics = [
+          :visitors,
+          :pageviews,
+          :visits,
+          :views_per_visit,
+          :bounce_rate,
+          :visit_duration
+        ]
 
-    {metrics, column_headers}
-  end
-
-  defp csv_graph_metrics(_) do
-    metrics = [:visitors, :pageviews, :visits, :views_per_visit, :bounce_rate, :visit_duration]
-    column_headers = [:date | metrics]
+        {
+          metrics,
+          [:date | metrics]
+        }
+      end
 
     {metrics, column_headers}
   end

test/plausible/google/search_console/filters_test.exs

@@ -3,9 +3,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
   use Plausible.DataCase, async: true
 
   test "transforms simple page filter" do
-    filters = %{
-      "visit:entry_page" => {:is, "/page"}
-    }
+    filters = [
+      [:is, "visit:entry_page", "/page"]
+    ]
 
     {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
 
@@ -15,9 +15,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
   end
 
   test "transforms matches page filter" do
-    filters = %{
-      "visit:entry_page" => {:matches, "*page*"}
-    }
+    filters = [
+      [:matches, "visit:entry_page", "*page*"]
+    ]
 
     {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
 
@@ -35,9 +35,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
   end
 
   test "transforms member page filter" do
-    filters = %{
-      "visit:entry_page" => {:member, ["/pageA", "/pageB"]}
-    }
+    filters = [
+      [:member, "visit:entry_page", ["/pageA", "/pageB"]]
+    ]
 
     {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
 
@@ -55,9 +55,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
   end
 
   test "transforms matches_member page filter" do
-    filters = %{
-      "visit:entry_page" => {:matches_member, ["/pageA*", "/pageB*"]}
-    }
+    filters = [
+      [:matches_member, "visit:entry_page", ["/pageA*", "/pageB*"]]
+    ]
 
     {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
 
@@ -75,9 +75,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
   end
 
   test "transforms event:page exactly like visit:entry_page" do
-    filters = %{
-      "event:page" => {:matches_member, ["/pageA*", "/pageB*"]}
-    }
+    filters = [
+      [:matches_member, "event:page", ["/pageA*", "/pageB*"]]
+    ]
 
     {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
 
@@ -95,9 +95,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
   end
 
   test "transforms simple visit:screen filter" do
-    filters = %{
-      "visit:screen" => {:is, "Desktop"}
-    }
+    filters = [
+      [:is, "visit:screen", "Desktop"]
+    ]
 
     {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
 
@@ -105,9 +105,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
   end
 
   test "transforms member visit:screen filter" do
-    filters = %{
-      "visit:screen" => {:member, ["Mobile", "Tablet"]}
-    }
+    filters = [
+      [:member, "visit:screen", ["Mobile", "Tablet"]]
+    ]
 
     {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
 
@@ -121,9 +121,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
   end
 
   test "transforms simple visit:country filter to alpha3" do
-    filters = %{
-      "visit:country" => {:is, "EE"}
-    }
+    filters = [
+      [:is, "visit:country", "EE"]
+    ]
 
     {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
 
@@ -131,9 +131,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
   end
 
   test "transforms member visit:country filter" do
-    filters = %{
-      "visit:country" => {:member, ["EE", "PL"]}
-    }
+    filters = [
+      [:member, "visit:country", ["EE", "PL"]]
+    ]
 
     {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
 
@@ -147,11 +147,11 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
   end
 
   test "filters can be combined" do
-    filters = %{
-      "visit:entry_page" => {:matches, "*web-analytics*"},
-      "visit:screen" => {:is, "Desktop"},
-      "visit:country" => {:member, ["EE", "PL"]}
-    }
+    filters = [
+      [:member, "visit:country", ["EE", "PL"]],
+      [:matches, "visit:entry_page", "*web-analytics*"],
+      [:is, "visit:screen", "Desktop"]
+    ]
 
     {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
 
@@ -171,12 +171,12 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
   end
 
   test "when unsupported filter is included the whole set becomes invalid" do
-    filters = %{
-      "visit:entry_page" => {:matches, "*web-analytics*"},
-      "visit:screen" => {:is, "Desktop"},
-      "visit:country" => {:member, ["EE", "PL"]},
-      "visit:utm_medium" => {:is, "facebook"}
-    }
+    filters = [
+      [:matches, "visit:entry_page", "*web-analytics*"],
+      [:is, "visit:screen", "Desktop"],
+      [:member, "visit:country", ["EE", "PL"]],
+      [:is, "visit:utm_medium", "facebook"]
+    ]
 
     assert :unsupported_filters = Filters.transform("sc-domain:plausible.io", filters)
   end

test/plausible/stats/dashboard_filter_parser_test.exs

@@ -9,199 +9,199 @@ defmodule Plausible.Stats.DashboardFilterParserTest do
   describe "adding prefix" do
     test "adds appropriate prefix to filter" do
       %{"page" => "/"}
-      |> assert_parsed(%{"event:page" => {:is, "/"}})
+      |> assert_parsed([[:is, "event:page", "/"]])
 
       %{"goal" => "Signup"}
-      |> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}})
+      |> assert_parsed([[:is, "event:goal", {:event, "Signup"}]])
 
       %{"goal" => "Visit /blog"}
-      |> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}})
+      |> assert_parsed([[:is, "event:goal", {:page, "/blog"}]])
 
       %{"source" => "Google"}
-      |> assert_parsed(%{"visit:source" => {:is, "Google"}})
+      |> assert_parsed([[:is, "visit:source", "Google"]])
 
       %{"referrer" => "cnn.com"}
-      |> assert_parsed(%{"visit:referrer" => {:is, "cnn.com"}})
+      |> assert_parsed([[:is, "visit:referrer", "cnn.com"]])
 
       %{"utm_medium" => "search"}
-      |> assert_parsed(%{"visit:utm_medium" => {:is, "search"}})
+      |> assert_parsed([[:is, "visit:utm_medium", "search"]])
 
       %{"utm_source" => "bing"}
-      |> assert_parsed(%{"visit:utm_source" => {:is, "bing"}})
+      |> assert_parsed([[:is, "visit:utm_source", "bing"]])
 
       %{"utm_content" => "content"}
-      |> assert_parsed(%{"visit:utm_content" => {:is, "content"}})
+      |> assert_parsed([[:is, "visit:utm_content", "content"]])
 
       %{"utm_term" => "term"}
-      |> assert_parsed(%{"visit:utm_term" => {:is, "term"}})
+      |> assert_parsed([[:is, "visit:utm_term", "term"]])
 
       %{"screen" => "Desktop"}
-      |> assert_parsed(%{"visit:screen" => {:is, "Desktop"}})
+      |> assert_parsed([[:is, "visit:screen", "Desktop"]])
 
       %{"browser" => "Opera"}
-      |> assert_parsed(%{"visit:browser" => {:is, "Opera"}})
+      |> assert_parsed([[:is, "visit:browser", "Opera"]])
 
       %{"browser_version" => "10.1"}
-      |> assert_parsed(%{"visit:browser_version" => {:is, "10.1"}})
+      |> assert_parsed([[:is, "visit:browser_version", "10.1"]])
 
       %{"os" => "Linux"}
-      |> assert_parsed(%{"visit:os" => {:is, "Linux"}})
+      |> assert_parsed([[:is, "visit:os", "Linux"]])
 
       %{"os_version" => "13.0"}
-      |> assert_parsed(%{"visit:os_version" => {:is, "13.0"}})
+      |> assert_parsed([[:is, "visit:os_version", "13.0"]])
 
       %{"country" => "EE"}
-      |> assert_parsed(%{"visit:country" => {:is, "EE"}})
+      |> assert_parsed([[:is, "visit:country", "EE"]])
 
       %{"region" => "EE-12"}
-      |> assert_parsed(%{"visit:region" => {:is, "EE-12"}})
+      |> assert_parsed([[:is, "visit:region", "EE-12"]])
 
       %{"city" => "123"}
-      |> assert_parsed(%{"visit:city" => {:is, "123"}})
+      |> assert_parsed([[:is, "visit:city", "123"]])
 
       %{"entry_page" => "/blog"}
-      |> assert_parsed(%{"visit:entry_page" => {:is, "/blog"}})
+      |> assert_parsed([[:is, "visit:entry_page", "/blog"]])
 
       %{"exit_page" => "/blog"}
-      |> assert_parsed(%{"visit:exit_page" => {:is, "/blog"}})
+      |> assert_parsed([[:is, "visit:exit_page", "/blog"]])
 
       %{"props" => %{"cta" => "Top"}}
-      |> assert_parsed(%{"event:props:cta" => {:is, "Top"}})
+      |> assert_parsed([[:is, "event:props:cta", "Top"]])
 
       %{"hostname" => "dummy.site"}
-      |> assert_parsed(%{"event:hostname" => {:is, "dummy.site"}})
+      |> assert_parsed([[:is, "event:hostname", "dummy.site"]])
     end
   end
 
   describe "escaping pipe character" do
     test "in simple is filter" do
       %{"goal" => ~S(Foo \| Bar)}
-      |> assert_parsed(%{"event:goal" => {:is, {:event, "Foo | Bar"}}})
+      |> assert_parsed([[:is, "event:goal", {:event, "Foo | Bar"}]])
     end
 
     test "in member filter" do
       %{"page" => ~S(/|\|)}
-      |> assert_parsed(%{"event:page" => {:member, ["/", "|"]}})
+      |> assert_parsed([[:member, "event:page", ["/", "|"]]])
     end
   end
 
   describe "is not filter type" do
     test "simple is not filter" do
       %{"page" => "!/"}
-      |> assert_parsed(%{"event:page" => {:is_not, "/"}})
+      |> assert_parsed([[:is_not, "event:page", "/"]])
 
       %{"props" => %{"cta" => "!Top"}}
-      |> assert_parsed(%{"event:props:cta" => {:is_not, "Top"}})
+      |> assert_parsed([[:is_not, "event:props:cta", "Top"]])
     end
   end
 
   describe "member filter type" do
     test "simple member filter" do
       %{"page" => "/|/blog"}
-      |> assert_parsed(%{"event:page" => {:member, ["/", "/blog"]}})
+      |> assert_parsed([[:member, "event:page", ["/", "/blog"]]])
     end
 
     test "escaping pipe character" do
       %{"page" => "/|\\|"}
-      |> assert_parsed(%{"event:page" => {:member, ["/", "|"]}})
+      |> assert_parsed([[:member, "event:page", ["/", "|"]]])
     end
 
     test "mixed goals" do
       %{"goal" => "Signup|Visit /thank-you"}
-      |> assert_parsed(%{"event:goal" => {:member, [{:event, "Signup"}, {:page, "/thank-you"}]}})
+      |> assert_parsed([[:member, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]])
 
       %{"goal" => "Visit /thank-you|Signup"}
-      |> assert_parsed(%{"event:goal" => {:member, [{:page, "/thank-you"}, {:event, "Signup"}]}})
+      |> assert_parsed([[:member, "event:goal", [{:page, "/thank-you"}, {:event, "Signup"}]]])
     end
   end
 
   describe "matches_member filter type" do
     test "parses matches_member filter type" do
       %{"page" => "/|/blog**"}
-      |> assert_parsed(%{"event:page" => {:matches_member, ["/", "/blog**"]}})
+      |> assert_parsed([[:matches_member, "event:page", ["/", "/blog**"]]])
     end
 
     test "parses not_matches_member filter type" do
       %{"page" => "!/|/blog**"}
-      |> assert_parsed(%{"event:page" => {:not_matches_member, ["/", "/blog**"]}})
+      |> assert_parsed([[:not_matches_member, "event:page", ["/", "/blog**"]]])
     end
   end
 
   describe "contains filter type" do
     test "single contains" do
       %{"page" => "~blog"}
-      |> assert_parsed(%{"event:page" => {:matches, "**blog**"}})
+      |> assert_parsed([[:matches, "event:page", "**blog**"]])
     end
 
     test "negated contains" do
       %{"page" => "!~articles"}
-      |> assert_parsed(%{"event:page" => {:does_not_match, "**articles**"}})
+      |> assert_parsed([[:does_not_match, "event:page", "**articles**"]])
     end
 
     test "contains member" do
       %{"page" => "~articles|blog"}
-      |> assert_parsed(%{"event:page" => {:matches_member, ["**articles**", "**blog**"]}})
+      |> assert_parsed([[:matches_member, "event:page", ["**articles**", "**blog**"]]])
     end
 
     test "not contains member" do
       %{"page" => "!~articles|blog"}
-      |> assert_parsed(%{"event:page" => {:not_matches_member, ["**articles**", "**blog**"]}})
+      |> assert_parsed([[:not_matches_member, "event:page", ["**articles**", "**blog**"]]])
     end
   end
 
   describe "not_member filter type" do
     test "simple not_member filter" do
       %{"page" => "!/|/blog"}
-      |> assert_parsed(%{"event:page" => {:not_member, ["/", "/blog"]}})
+      |> assert_parsed([[:not_member, "event:page", ["/", "/blog"]]])
     end
 
     test "mixed goals" do
       %{"goal" => "!Signup|Visit /thank-you"}
-      |> assert_parsed(%{
-        "event:goal" => {:not_member, [{:event, "Signup"}, {:page, "/thank-you"}]}
-      })
+      |> assert_parsed([
+        [:not_member, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]
+      ])
 
       %{"goal" => "!Visit /thank-you|Signup"}
-      |> assert_parsed(%{
-        "event:goal" => {:not_member, [{:page, "/thank-you"}, {:event, "Signup"}]}
-      })
+      |> assert_parsed([
+        [:not_member, "event:goal", [{:page, "/thank-you"}, {:event, "Signup"}]]
+      ])
     end
   end
 
   describe "matches filter type" do
     test "can be used with `goal` or `page` filters" do
       %{"page" => "/blog/post-*"}
-      |> assert_parsed(%{"event:page" => {:matches, "/blog/post-*"}})
+      |> assert_parsed([[:matches, "event:page", "/blog/post-*"]])
 
       %{"goal" => "Visit /blog/post-*"}
-      |> assert_parsed(%{"event:goal" => {:matches, {:page, "/blog/post-*"}}})
+      |> assert_parsed([[:matches, "event:goal", {:page, "/blog/post-*"}]])
     end
 
     test "other filters default to `is` even when wildcard is present" do
       %{"country" => "Germa**"}
-      |> assert_parsed(%{"visit:country" => {:is, "Germa**"}})
+      |> assert_parsed([[:is, "visit:country", "Germa**"]])
     end
   end
 
   describe "does_not_match filter type" do
     test "can be used with `page` filter" do
       %{"page" => "!/blog/post-*"}
-      |> assert_parsed(%{"event:page" => {:does_not_match, "/blog/post-*"}})
+      |> assert_parsed([[:does_not_match, "event:page", "/blog/post-*"]])
     end
 
     test "other filters default to is_not even when wildcard is present" do
       %{"country" => "!Germa**"}
-      |> assert_parsed(%{"visit:country" => {:is_not, "Germa**"}})
+      |> assert_parsed([[:is_not, "visit:country", "Germa**"]])
     end
   end
 
   describe "contains prefix filter type" do
     test "can be used with any filter" do
       %{"page" => "~/blog/post"}
-      |> assert_parsed(%{"event:page" => {:matches, "**/blog/post**"}})
+      |> assert_parsed([[:matches, "event:page", "**/blog/post**"]])
 
       %{"source" => "~facebook"}
-      |> assert_parsed(%{"visit:source" => {:matches, "**facebook**"}})
+      |> assert_parsed([[:matches, "visit:source", "**facebook**"]])
     end
   end
 end

test/plausible/stats/filters_test.exs

@@ -12,90 +12,90 @@ defmodule Plausible.Stats.FiltersTest do
   describe "parses filter expression" do
     test "simple positive" do
       "event:name==pageview"
-      |> assert_parsed(%{"event:name" => {:is, "pageview"}})
+      |> assert_parsed([[:is, "event:name", "pageview"]])
     end
 
     test "simple negative" do
       "event:name!=pageview"
-      |> assert_parsed(%{"event:name" => {:is_not, "pageview"}})
+      |> assert_parsed([[:is_not, "event:name", "pageview"]])
     end
 
     test "whitespace is trimmed" do
       " event:name == pageview "
-      |> assert_parsed(%{"event:name" => {:is, "pageview"}})
+      |> assert_parsed([[:is, "event:name", "pageview"]])
     end
 
     test "wildcard" do
       "event:page==/blog/post-*"
-      |> assert_parsed(%{"event:page" => {:matches, "/blog/post-*"}})
+      |> assert_parsed([[:matches, "event:page", "/blog/post-*"]])
     end
 
     test "negative wildcard" do
       "event:page!=/blog/post-*"
-      |> assert_parsed(%{"event:page" => {:does_not_match, "/blog/post-*"}})
+      |> assert_parsed([[:does_not_match, "event:page", "/blog/post-*"]])
     end
 
     test "custom event goal" do
       "event:goal==Signup"
-      |> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}})
+      |> assert_parsed([[:is, "event:goal", {:event, "Signup"}]])
     end
 
     test "pageview goal" do
       "event:goal==Visit /blog"
-      |> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}})
+      |> assert_parsed([[:is, "event:goal", {:page, "/blog"}]])
     end
 
     test "member" do
       "visit:country==FR|GB|DE"
-      |> assert_parsed(%{"visit:country" => {:member, ["FR", "GB", "DE"]}})
+      |> assert_parsed([[:member, "visit:country", ["FR", "GB", "DE"]]])
     end
 
     test "member + wildcard" do
       "event:page==/blog**|/newsletter|/*/"
-      |> assert_parsed(%{"event:page" => {:matches, "/blog**|/newsletter|/*/"}})
+      |> assert_parsed([[:matches, "event:page", "/blog**|/newsletter|/*/"]])
     end
 
     test "combined with \";\"" do
       "event:page==/blog**|/newsletter|/*/ ; visit:country==FR|GB|DE"
-      |> assert_parsed(%{
-        "event:page" => {:matches, "/blog**|/newsletter|/*/"},
-        "visit:country" => {:member, ["FR", "GB", "DE"]}
-      })
+      |> assert_parsed([
+        [:matches, "event:page", "/blog**|/newsletter|/*/"],
+        [:member, "visit:country", ["FR", "GB", "DE"]]
+      ])
     end
 
     test "escaping pipe character" do
       "utm_campaign==campaign \\| 1"
-      |> assert_parsed(%{"utm_campaign" => {:is, "campaign | 1"}})
+      |> assert_parsed([[:is, "utm_campaign", "campaign | 1"]])
     end
 
     test "escaping pipe character in member filter" do
       "utm_campaign==campaign \\| 1|campaign \\| 2"
-      |> assert_parsed(%{"utm_campaign" => {:member, ["campaign | 1", "campaign | 2"]}})
+      |> assert_parsed([[:member, "utm_campaign", ["campaign | 1", "campaign | 2"]]])
     end
 
     test "keeps escape characters in member + wildcard filter" do
       "event:page==/**\\|page|/other/page"
-      |> assert_parsed(%{"event:page" => {:matches, "/**\\|page|/other/page"}})
+      |> assert_parsed([[:matches, "event:page", "/**\\|page|/other/page"]])
     end
 
     test "gracefully fails to parse garbage" do
       "bfg10309\uff1cs1\ufe65s2\u02bas3\u02b9hjl10309"
-      |> assert_parsed(%{})
+      |> assert_parsed([])
     end
 
     test "gracefully fails to parse garbage with double quotes" do
       "\";print(md5(31337));$a=\""
-      |> assert_parsed(%{})
+      |> assert_parsed([])
     end
 
     test "gracefully fails to parse garbage country code" do
       "visit:country==AKSJSDFKJSS"
-      |> assert_parsed(%{})
+      |> assert_parsed([])
     end
 
     test "gracefully fails to parse garbage country code (with pipes)" do
       "visit:country==ET'||DBMS_PIPE.RECEIVE_MESSAGE(CHR(98)||CHR(98)||CHR(98),15)||'"
-      |> assert_parsed(%{})
+      |> assert_parsed([])
     end
   end
 end

test/plausible/stats/query_test.exs

@@ -194,14 +194,14 @@ defmodule Plausible.Stats.QueryTest do
       filters = Jason.encode!(%{"goal" => "Signup"})
       q = Query.from(site, %{"period" => "6mo", "filters" => filters})
 
-      assert q.filters["event:goal"] == {:is, {:event, "Signup"}}
+      assert q.filters == [[:is, "event:goal", {:event, "Signup"}]]
     end
 
     test "parses source filter", %{site: site} do
       filters = Jason.encode!(%{"source" => "Twitter"})
       q = Query.from(site, %{"period" => "6mo", "filters" => filters})
 
-      assert q.filters["visit:source"] == {:is, "Twitter"}
+      assert q.filters == [[:is, "visit:source", "Twitter"]]
     end
   end