Export and import custom events via CSV (#4096)

* Export and import custom events via CSV

* Add prop support of url for cloaked links and path for 404s in imported queries

* Handle custom events with empty URL and path properties gracefully

* Make events with properties logic DRY and fix missed cloaked link

* Add test for path property breakdown

* Update raw CH data fixture and extend CSV importer tests

* Fix broken query condition after rebase

* Update CHANGELOG.md

CHANGELOG.md

@@ -45,6 +45,7 @@ All notable changes to this project will be documented in this file.
 - Import custom events from Google Analytics 4
 - Ability to filter Search Console keywords by page, country and device plausible/analytics#4077
 - Add `DATA_DIR` env var for exports/imports plausible/analytics#4100
+- Add custom events support to CSV export and import
 
 ### Removed
 - Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversions

lib/plausible/exports.ex

@@ -4,6 +4,7 @@ defmodule Plausible.Exports do
   """
 
   use Plausible
+  use Plausible.Stats.Fragments
   import Ecto.Query
 
   @doc "Schedules CSV export job to S3 storage"
@@ -234,6 +235,8 @@ defmodule Plausible.Exports do
       filename.("imported_pages") => export_pages_q(site_id, timezone, date_range),
       filename.("imported_entry_pages") => export_entry_pages_q(site_id, timezone, date_range),
       filename.("imported_exit_pages") => export_exit_pages_q(site_id, timezone, date_range),
+      filename.("imported_custom_events") =>
+        export_custom_events_q(site_id, timezone, date_range),
       filename.("imported_locations") => export_locations_q(site_id, timezone, date_range),
       filename.("imported_devices") => export_devices_q(site_id, timezone, date_range),
       filename.("imported_browsers") => export_browsers_q(site_id, timezone, date_range),
@@ -465,6 +468,43 @@ defmodule Plausible.Exports do
       ]
   end
 
+  defp export_custom_events_q(site_id, timezone, date_range) do
+    from e in sampled("events_v2"),
+      where: ^export_filter(site_id, date_range),
+      where: e.name != "pageview",
+      group_by: [
+        selected_as(:date),
+        e.name,
+        selected_as(:link_url),
+        selected_as(:path)
+      ],
+      order_by: selected_as(:date),
+      select: [
+        date(e.timestamp, ^timezone),
+        e.name,
+        selected_as(
+          fragment(
+            "if(? in ?, ?, '')",
+            e.name,
+            ^Plausible.Imported.goals_with_url(),
+            get_by_key(e, :meta, "url")
+          ),
+          :link_url
+        ),
+        selected_as(
+          fragment(
+            "if(? in ?, ?, '')",
+            e.name,
+            ^Plausible.Imported.goals_with_path(),
+            get_by_key(e, :meta, "path")
+          ),
+          :path
+        ),
+        visitors(e),
+        selected_as(fragment("toUInt64(round(count()*any(_sample_factor)))"), :events)
+      ]
+  end
+
   defp export_locations_q(site_id, timezone, date_range) do
     from s in sampled("sessions_v2"),
       where: ^export_filter(site_id, date_range),

lib/plausible/imported.ex

@@ -38,6 +38,11 @@ defmodule Plausible.Imported do
   # Maximum number of complete imports to account for when querying stats
   @max_complete_imports 5
 
+  # Goals which can be filtered by url property
+  @goals_with_url ["Outbound Link: Click", "Cloaked Link: Click", "File Download"]
+  # Goals which can be filtered by path property
+  @goals_with_path ["404"]
+
   @spec schemas() :: [module()]
   def schemas, do: @tables
 
@@ -49,6 +54,16 @@ defmodule Plausible.Imported do
     @max_complete_imports
   end
 
+  @spec goals_with_url() :: [String.t()]
+  def goals_with_url() do
+    @goals_with_url
+  end
+
+  @spec goals_with_path() :: [String.t()]
+  def goals_with_path() do
+    @goals_with_path
+  end
+
   @spec load_import_data(Site.t()) :: Site.t()
   def load_import_data(%{import_data_loaded: true} = site), do: site
 

lib/plausible/imported/csv_importer.ex

@@ -154,6 +154,8 @@ defmodule Plausible.Imported.CSVImporter do
       "date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
     "imported_exit_pages" =>
       "date Date, exit_page String, visitors UInt64, visit_duration UInt64, exits UInt64, bounces UInt32, pageviews UInt64",
+    "imported_custom_events" =>
+      "date Date, name String, link_url String, path String, visitors UInt64, events UInt64",
     "imported_locations" =>
       "date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
     "imported_operating_systems" =>

lib/plausible/imported/custom_event.ex

@@ -9,6 +9,7 @@ defmodule Plausible.Imported.CustomEvent do
     field :date, :date
     field :name, :string
     field :link_url, :string
+    field :path, :string
     field :visitors, Ch, type: "UInt64"
     field :events, Ch, type: "UInt64"
   end

lib/plausible/stats/imported.ex

@@ -7,6 +7,7 @@ defmodule Plausible.Stats.Imported do
 
   @no_ref "Direct / None"
   @not_set "(not set)"
+  @none "(none)"
 
   @property_to_table_mappings %{
     "visit:source" => "imported_sources",
@@ -28,12 +29,14 @@ defmodule Plausible.Stats.Imported do
     "visit:os_version" => "imported_operating_systems",
     "event:page" => "imported_pages",
     "event:name" => "imported_custom_events",
-    "event:props:url" => "imported_custom_events"
+    "event:props:url" => "imported_custom_events",
+    "event:props:path" => "imported_custom_events"
   }
 
   @imported_properties Map.keys(@property_to_table_mappings)
 
-  @goals_with_url ["Outbound Link: Click", "Cloaked Link: Click", "File Download"]
+  @goals_with_url Plausible.Imported.goals_with_url()
+  @goals_with_path Plausible.Imported.goals_with_path()
 
   @doc """
   Returns a boolean indicating whether the combination of filters and
@@ -52,6 +55,7 @@ defmodule Plausible.Stats.Imported do
       {0, "event:props:" <> _} -> false
       {0, _} -> true
       {1, "event:props:url"} -> has_special_goal_filter?(query, @goals_with_url)
+      {1, "event:props:path"} -> has_special_goal_filter?(query, @goals_with_path)
       {_, _} -> false
     end
   end
@@ -126,7 +130,7 @@ defmodule Plausible.Stats.Imported do
 
     join_on =
       case dim do
-        :url ->
+        _ when dim in [:url, :path] ->
           dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value)
 
         :os_version ->
@@ -224,6 +228,14 @@ defmodule Plausible.Stats.Imported do
     end
   end
 
+  defp maybe_apply_filter(q, query, "event:props:path", _) do
+    if name = find_special_goal_filter(query, @goals_with_path) do
+      where(q, [i], i.name == ^name)
+    else
+      q
+    end
+  end
+
   defp maybe_apply_filter(q, query, property, dim) do
     case Query.get_filter(query, property) do
       [:member, _, list] -> where(q, [i], field(i, ^dim) in ^list)
@@ -469,7 +481,17 @@ defmodule Plausible.Stats.Imported do
   defp group_imported_by(q, :url) do
     q
     |> group_by([i], i.link_url)
-    |> select_merge([i], %{breakdown_prop_value: i.link_url})
+    |> select_merge([i], %{
+      breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
+    })
+  end
+
+  defp group_imported_by(q, :path) do
+    q
+    |> group_by([i], i.path)
+    |> select_merge([i], %{
+      breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
+    })
   end
 
   defp select_joined_dimension(q, :city) do
@@ -493,7 +515,7 @@ defmodule Plausible.Stats.Imported do
     })
   end
 
-  defp select_joined_dimension(q, :url) do
+  defp select_joined_dimension(q, dim) when dim in [:url, :path] do
     select_merge(q, [s, i], %{
       breakdown_prop_value:
         fragment(

test/plausible/exports_test.exs

@@ -12,6 +12,7 @@ defmodule Plausible.ExportsTest do
 
       assert Map.keys(queries) == [
                "imported_browsers.csv",
+               "imported_custom_events.csv",
                "imported_devices.csv",
                "imported_entry_pages.csv",
                "imported_exit_pages.csv",
@@ -31,6 +32,7 @@ defmodule Plausible.ExportsTest do
 
       assert Map.keys(queries) == [
                "imported_browsers_20230101_20240312.csv",
+               "imported_custom_events_20230101_20240312.csv",
                "imported_devices_20230101_20240312.csv",
                "imported_entry_pages_20230101_20240312.csv",
                "imported_exit_pages_20230101_20240312.csv",
@@ -50,6 +52,7 @@ defmodule Plausible.ExportsTest do
 
       assert Map.keys(queries) == [
                "imported_browsers.ch",
+               "imported_custom_events.ch",
                "imported_devices.ch",
                "imported_entry_pages.ch",
                "imported_exit_pages.ch",

test/plausible/imported/csv_importer_test.exs

@@ -18,6 +18,7 @@ defmodule Plausible.Imported.CSVImporterTest do
     test "parses job args properly", %{user: user, site: site} do
       tables = [
         "imported_browsers",
+        "imported_custom_events",
         "imported_devices",
         "imported_entry_pages",
         "imported_exit_pages",
@@ -189,6 +190,19 @@ defmodule Plausible.Imported.CSVImporterTest do
           "2021-12-31",1,1,"/1710995203264225236"
           """
         },
+        %{
+          name: "imported_custom_events_20211230_20211231.csv",
+          body: """
+          "date","name","link_url","path","visitors","events"
+          "2021-12-30","Filter Menu: Open","","",300,1652
+          "2021-12-30","Signup","","",40,82
+          "2021-12-30","Signup via invitation","","",5,10
+          "2021-12-31","Signup via invitation","","",5,10
+          "2021-12-31","Signup","","",39,78
+          "2021-12-31","Filter Menu: Open","","",295,1394
+          "2021-12-31","Newsletter signup","","",1,2
+          """
+        },
         %{
           name: "imported_locations_20211230_20211231.csv",
           body: """
@@ -421,6 +435,13 @@ defmodule Plausible.Imported.CSVImporterTest do
       exported_site = insert(:site, members: [user])
       imported_site = insert(:site, members: [user])
 
+      insert(:goal, site: exported_site, event_name: "Outbound Link: Click")
+      insert(:goal, site: exported_site, event_name: "404")
+      insert(:goal, site: exported_site, event_name: "click-button")
+      insert(:goal, site: imported_site, event_name: "Outbound Link: Click")
+      insert(:goal, site: imported_site, event_name: "404")
+      insert(:goal, site: imported_site, event_name: "click-button")
+
       process_csv = fn path ->
         [header | rows] = NimbleCSV.RFC4180.parse_string(File.read!(path), skip_headers: false)
 
@@ -438,12 +459,12 @@ defmodule Plausible.Imported.CSVImporterTest do
 
       Plausible.IngestRepo.query!([
         "insert into events_v2 format CSVWithNames\n",
-        process_csv.("fixture/plausible_io_events_v2_2024_03_01_2024_03_31_500users_dump.csv")
+        process_csv.("fixture/markosaric_com_events_v2_2024_04_01_2024_04_30_dump.csv")
       ])
 
       Plausible.IngestRepo.query!([
         "insert into sessions_v2 format CSVWithNames\n",
-        process_csv.("fixture/plausible_io_sessions_v2_2024_03_01_2024_03_31_500users_dump.csv")
+        process_csv.("fixture/markosaric_com_sessions_v2_2024_04_01_2024_04_30_dump.csv")
       ])
 
       # export archive to s3
@@ -511,14 +532,14 @@ defmodule Plausible.Imported.CSVImporterTest do
 
       # validate import
       assert %SiteImport{
-               start_date: ~D[2024-03-28],
+               start_date: ~D[2024-04-01],
-               end_date: ~D[2024-03-31],
+               end_date: ~D[2024-04-30],
                source: :csv,
                status: :completed
              } = Repo.get_by!(SiteImport, site_id: imported_site.id)
 
       assert Plausible.Stats.Clickhouse.imported_pageview_count(exported_site) == 0
-      assert Plausible.Stats.Clickhouse.imported_pageview_count(imported_site) == 6298
+      assert Plausible.Stats.Clickhouse.imported_pageview_count(imported_site) == 1745
 
       # compare original and imported data via stats api requests
       results = fn path, params ->
@@ -535,25 +556,40 @@ defmodule Plausible.Imported.CSVImporterTest do
         %{
           "site_id" => site.domain,
           "period" => "custom",
-          "date" => "2024-03-28,2024-03-31",
+          "date" => "2024-04-01,2024-04-30",
           "with_imported" => true
         }
       end
 
-      breakdown = fn params_or_site, by ->
+      breakdown = fn params_or_site, by, opts ->
+        metrics =
+          Keyword.get(opts, :metrics, "visitors,visits,pageviews,visit_duration,bounce_rate")
+
+        filters = Keyword.get(opts, :filters, "")
+
+        by_without_prefix =
+          by
+          |> String.trim_leading("event:")
+          |> String.trim_leading("visit:")
+          |> String.trim_leading("props:")
+
         params =
           case params_or_site do
             %Plausible.Site{} = site ->
               common_params.(site)
-              |> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate")
+              |> Map.put("metrics", metrics)
+              |> Map.put("filters", filters)
               |> Map.put("limit", 1000)
-              |> Map.put("property", "visit:#{by}")
+              |> Map.put("property", by)
 
             params ->
               params
           end
 
-        Enum.sort_by(results.("/api/v1/stats/breakdown", params), &Map.fetch!(&1, by))
+        Enum.sort_by(
+          results.("/api/v1/stats/breakdown", params),
+          &Map.fetch!(&1, by_without_prefix)
+        )
       end
 
       # timeseries
@@ -572,39 +608,23 @@ defmodule Plausible.Imported.CSVImporterTest do
         assert exported["date"] == imported["date"]
         assert exported["pageviews"] == imported["pageviews"]
         assert exported["bounce_rate"] == imported["bounce_rate"]
-        assert exported["visitors"] == imported["visitors"]
+        assert_in_delta exported["visitors"], imported["visitors"], 1
         assert exported["visits"] == imported["visits"]
         assert_in_delta exported["visit_duration"], imported["visit_duration"], 1
       end)
 
       # timeseries' views per visit difference is within 3%
-      assert summary(field(exported_timeseries, "views_per_visit")) == [
+      assert summary(field(exported_timeseries, "views_per_visit")) ==
-               2.96,
+               [0.98, 1.2025, 1.29, 1.3699999999999999, 1.64]
-               2.99,
-               3.065,
-               3.135,
-               3.15
-             ]
 
-      assert summary(field(imported_timeseries, "views_per_visit")) == [
+      assert summary(field(imported_timeseries, "views_per_visit")) ==
-               2.95,
+               [0.98, 1.1925, 1.29, 1.375, 1.64]
-               3.04,
-               3.075,
-               3.1025,
-               3.17
-             ]
 
       assert summary(
                pairwise(exported_timeseries, imported_timeseries, fn exported, imported ->
                  abs(1 - imported["views_per_visit"] / exported["views_per_visit"])
                end)
-             ) == [
+             ) == [0.0, 0.0, 0.0, 0.0, 0.0325203252032521]
-               0.0033783783783782884,
-               0.005606499356499317,
-               0.011161823621887501,
-               0.017814164004259808,
-               0.023333333333333206
-             ]
 
       # pages
       pages_params = fn site ->
@@ -614,8 +634,8 @@ defmodule Plausible.Imported.CSVImporterTest do
         |> Map.put("property", "event:page")
       end
 
-      exported_pages = breakdown.(pages_params.(exported_site), "page")
+      exported_pages = breakdown.(pages_params.(exported_site), "visit:page", [])
-      imported_pages = breakdown.(pages_params.(imported_site), "page")
+      imported_pages = breakdown.(pages_params.(imported_site), "visit:page", [])
 
       pairwise(exported_pages, imported_pages, fn exported, imported ->
         assert exported["page"] == imported["page"]
@@ -626,9 +646,9 @@ defmodule Plausible.Imported.CSVImporterTest do
         assert imported["time_on_page"] == 0
       end)
 
-      # page breakdown's visit_duration difference is within 1%
+      # NOTE: page breakdown's visitors difference is up to almost 43%
-      assert summary(field(exported_pages, "visit_duration")) == [0, 0, 25, 217.5, 743]
+      assert summary(field(exported_pages, "visit_duration")) == [0, 0.0, 13.0, 46.5, 264]
-      assert summary(field(imported_pages, "visit_duration")) == [0, 0, 25, 217.55, 742.8]
+      assert summary(field(imported_pages, "visit_duration")) == [0.0, 0.0, 13.3, 46.6, 264.5]
 
       assert summary(
                pairwise(exported_pages, imported_pages, fn exported, imported ->
@@ -643,11 +663,11 @@ defmodule Plausible.Imported.CSVImporterTest do
                    _no_diff = 0
                  end
                end)
-             ) == [0, 0, 0, 0, 0.002375296912114022]
+             ) == [0.0, 0.0, 0.0, 0.008812019016100597, 0.4285714285714286]
 
-      # NOTE: page breakdown's visitors difference is up to almost 37%
+      # NOTE: page breakdown's visitors difference is up to 28%
-      assert summary(field(exported_pages, "visitors")) == [1, 1, 2, 2.5, 393]
+      assert summary(field(exported_pages, "visitors")) == [1, 1.0, 5.5, 17.75, 495]
-      assert summary(field(imported_pages, "visitors")) == [1, 1, 2, 2.5, 617]
+      assert summary(field(imported_pages, "visitors")) == [1, 1.75, 5.5, 20.0, 511]
 
       assert summary(
                pairwise(exported_pages, imported_pages, fn exported, imported ->
@@ -657,11 +677,11 @@ defmodule Plausible.Imported.CSVImporterTest do
                  # only consider non tiny readings
                  if e > 5, do: abs(1 - e / i), else: 0
                end)
-             ) == [0, 0, 0, 0, 0.36304700162074555]
+             ) == [0, 0.0, 0.0, 0.0, 0.2857142857142857]
 
-      # page breakdown's visits difference is within 2% for non-tiny values
+      # page breakdown's visits difference is within 6% for non-tiny values
-      assert summary(field(exported_pages, "visits")) == [1, 1, 2, 3, 1774]
+      assert summary(field(exported_pages, "visits")) == [1, 1.75, 5.5, 20.25, 526]
-      assert summary(field(imported_pages, "visits")) == [1, 1, 2, 2.5, 1777]
+      assert summary(field(imported_pages, "visits")) == [1, 1.75, 5.5, 20.25, 527]
 
       assert summary(
                pairwise(exported_pages, imported_pages, fn exported, imported ->
@@ -671,11 +691,11 @@ defmodule Plausible.Imported.CSVImporterTest do
                  # only consider non tiny readings
                  if e > 4, do: abs(1 - e / i), else: 0
                end)
-             ) == [0, 0, 0, 0, 0.01666666666666672]
+             ) == [0, 0.0, 0.0, 0.0, 0.06666666666666665]
 
       # sources
-      exported_sources = breakdown.(exported_site, "source")
+      exported_sources = breakdown.(exported_site, "visit:source", [])
-      imported_sources = breakdown.(imported_site, "source")
+      imported_sources = breakdown.(imported_site, "visit:source", [])
 
       pairwise(exported_sources, imported_sources, fn exported, imported ->
         assert exported["source"] == imported["source"]
@@ -685,22 +705,23 @@ defmodule Plausible.Imported.CSVImporterTest do
         assert_in_delta exported["visit_duration"], imported["visit_duration"], 1
       end)
 
-      # NOTE: source breakdown's visitors difference is up to almost 40%
+      # NOTE: source breakdown's visitors difference is up to 12%
-      assert summary(field(exported_sources, "visitors")) == [1, 1, 1, 2, 451]
+      assert summary(field(exported_sources, "visitors")) == [1, 1.0, 1.0, 7.25, 582]
-      assert summary(field(imported_sources, "visitors")) == [1, 1, 1, 2, 711]
+      assert summary(field(imported_sources, "visitors")) == [1, 1.0, 1.0, 7.25, 587]
 
       assert summary(
                pairwise(exported_sources, imported_sources, fn exported, imported ->
                  abs(1 - exported["visitors"] / imported["visitors"])
                end)
-             ) == [0, 0, 0, 0, 0.3656821378340366]
+             ) == [0.0, 0.0, 0.0, 0.0, 0.11111111111111116]
 
       # utm mediums
-      assert breakdown.(exported_site, "utm_medium") == breakdown.(imported_site, "utm_medium")
+      assert breakdown.(exported_site, "visit:utm_medium", []) ==
+               breakdown.(imported_site, "visit:utm_medium", [])
 
       # entry pages
-      exported_entry_pages = breakdown.(exported_site, "entry_page")
+      exported_entry_pages = breakdown.(exported_site, "visit:entry_page", [])
-      imported_entry_pages = breakdown.(imported_site, "entry_page")
+      imported_entry_pages = breakdown.(imported_site, "visit:entry_page", [])
 
       pairwise(exported_entry_pages, imported_entry_pages, fn exported, imported ->
         assert exported["entry_page"] == imported["entry_page"]
@@ -711,8 +732,8 @@ defmodule Plausible.Imported.CSVImporterTest do
       end)
 
       # NOTE: entry page breakdown's visitors difference is up to almost 50%
-      assert summary(field(exported_entry_pages, "visitors")) == [1, 1, 1, 2, 310]
+      assert summary(field(exported_entry_pages, "visitors")) == [1, 1.0, 5.0, 12.5, 458]
-      assert summary(field(imported_entry_pages, "visitors")) == [1, 1, 1, 2, 475]
+      assert summary(field(imported_entry_pages, "visitors")) == [1, 1.0, 5.0, 12.5, 473]
 
       assert summary(
                pairwise(exported_entry_pages, imported_entry_pages, fn exported, imported ->
@@ -721,21 +742,20 @@ defmodule Plausible.Imported.CSVImporterTest do
              ) == [0, 0, 0, 0, 0.5]
 
       # cities
-      exported_cities = breakdown.(exported_site, "city")
+      exported_cities = breakdown.(exported_site, "visit:city", [])
-      imported_cities = breakdown.(imported_site, "city")
+      imported_cities = breakdown.(imported_site, "visit:city", [])
 
       pairwise(exported_cities, imported_cities, fn exported, imported ->
         assert exported["city"] == imported["city"]
-        assert exported["bounce_rate"] == imported["bounce_rate"]
+        assert_in_delta exported["bounce_rate"], imported["bounce_rate"], 4
         assert exported["pageviews"] == imported["pageviews"]
-        assert_in_delta exported["visit_duration"], imported["visit_duration"], 1
+        assert_in_delta exported["visit_duration"], imported["visit_duration"], 2
         assert_in_delta exported["visits"], imported["visits"], 1
       end)
 
-      # NOTE: city breakdown's visitors relative difference is up to 60%,
+      # NOTE: city breakdown's visitors relative difference is up to 27%
-      #       but the absolute difference is small
+      assert summary(field(exported_cities, "visitors")) == [1, 1.0, 1.0, 2.0, 22]
-      assert summary(field(exported_cities, "visitors")) == [1, 1, 1, 1, 7]
+      assert summary(field(imported_cities, "visitors")) == [1, 1.0, 1.0, 2.0, 30]
-      assert summary(field(imported_cities, "visitors")) == [1, 1, 1, 3, 13]
 
       assert summary(
                pairwise(exported_cities, imported_cities, fn exported, imported ->
@@ -745,11 +765,11 @@ defmodule Plausible.Imported.CSVImporterTest do
                  # only consider non tiny readings
                  if e > 3, do: abs(1 - e / i), else: 0
                end)
-             ) == [0, 0, 0, 0, 0.6]
+             ) == [0, 0.0, 0.0, 0.0, 0.2666666666666667]
 
       # devices
-      exported_devices = breakdown.(exported_site, "device")
+      exported_devices = breakdown.(exported_site, "visit:device", [])
-      imported_devices = breakdown.(imported_site, "device")
+      imported_devices = breakdown.(imported_site, "visit:device", [])
 
       pairwise(exported_devices, imported_devices, fn exported, imported ->
         assert exported["device"] == imported["device"]
@@ -759,25 +779,19 @@ defmodule Plausible.Imported.CSVImporterTest do
         assert_in_delta exported["visit_duration"], imported["visit_duration"], 1
       end)
 
-      # NOTE: device breakdown's visitors difference is between 30% and 40%
+      # NOTE: device breakdown's visitors difference is up to 3%
-      assert summary(field(exported_devices, "visitors")) == [216, 232.25, 248.5, 264.75, 281]
+      assert summary(field(exported_devices, "visitors")) == [3, 21.75, 242.0, 535.25, 773]
-      assert summary(field(imported_devices, "visitors")) == [304, 341.5, 379, 416.5, 454]
+      assert summary(field(imported_devices, "visitors")) == [3, 21.75, 249.5, 551.0, 791]
 
       assert summary(
                pairwise(exported_devices, imported_devices, fn exported, imported ->
                  abs(1 - exported["visitors"] / imported["visitors"])
                end)
-             ) == [
+             ) == [0.0, 0.0, 0.011378002528445008, 0.025028787232157956, 0.031847133757961776]
-               0.2894736842105263,
-               0.3123695803385115,
-               0.3352654764664966,
-               0.3581613725944818,
-               0.3810572687224669
-             ]
 
       # browsers
-      exported_browsers = breakdown.(exported_site, "browser")
+      exported_browsers = breakdown.(exported_site, "visit:browser", [])
-      imported_browsers = breakdown.(imported_site, "browser")
+      imported_browsers = breakdown.(imported_site, "visit:browser", [])
 
       pairwise(exported_browsers, imported_browsers, fn exported, imported ->
         assert exported["browser"] == imported["browser"]
@@ -787,25 +801,19 @@ defmodule Plausible.Imported.CSVImporterTest do
         assert_in_delta exported["visit_duration"], imported["visit_duration"], 1
       end)
 
-      # NOTE: browser breakdown's visitors difference is up to almost 70%
+      # NOTE: browser breakdown's visitors difference is up to 4%
-      assert summary(field(exported_browsers, "visitors")) == [1, 1, 10, 105, 274]
+      assert summary(field(exported_browsers, "visitors")) == [1, 1.5, 8.0, 34.0, 572]
-      assert summary(field(imported_browsers, "visitors")) == [1, 2, 18, 156.5, 422]
+      assert summary(field(imported_browsers, "visitors")) == [1, 1.5, 8.0, 34.0, 588]
 
       assert summary(
                pairwise(exported_browsers, imported_browsers, fn exported, imported ->
                  abs(1 - exported["visitors"] / imported["visitors"])
                end)
-             ) == [
+             ) == [0.0, 0.0, 0.0, 0.0, 0.037122969837587005]
-               0,
-               0.1422018348623853,
-               0.3507109004739336,
-               0.43801169590643274,
-               0.6666666666666667
-             ]
 
       # os
-      exported_os = breakdown.(exported_site, "os")
+      exported_os = breakdown.(exported_site, "visit:os", [])
-      imported_os = breakdown.(imported_site, "os")
+      imported_os = breakdown.(imported_site, "visit:os", [])
 
       pairwise(exported_os, imported_os, fn exported, imported ->
         assert exported["os"] == imported["os"]
@@ -815,25 +823,19 @@ defmodule Plausible.Imported.CSVImporterTest do
         assert_in_delta exported["visit_duration"], imported["visit_duration"], 1
       end)
 
-      # NOTE: os breakdown's visitors difference is between 20% and 60%
+      # NOTE: os breakdown's visitors difference is up to 5%
-      assert summary(field(exported_os, "visitors")) == [2, 9.5, 51, 130, 165]
+      assert summary(field(exported_os, "visitors")) == [3, 11.0, 56.0, 311.0, 390]
-      assert summary(field(imported_os, "visitors")) == [5, 12.5, 70, 200, 258]
+      assert summary(field(imported_os, "visitors")) == [3, 11.0, 57.0, 315.0, 403]
 
       assert summary(
                pairwise(exported_os, imported_os, fn exported, imported ->
                  abs(1 - exported["visitors"] / imported["visitors"])
                end)
-             ) == [
+             ) == [0.0, 0.0, 0.0, 0.01754385964912286, 0.045592705167173286]
-               0.1578947368421053,
-               0.28315018315018314,
-               0.36046511627906974,
-               0.463855421686747,
-               0.6
-             ]
 
       # os versions
-      exported_os_versions = breakdown.(exported_site, "os_version")
+      exported_os_versions = breakdown.(exported_site, "visit:os_version", [])
-      imported_os_versions = breakdown.(imported_site, "os_version")
+      imported_os_versions = breakdown.(imported_site, "visit:os_version", [])
 
       pairwise(exported_os_versions, imported_os_versions, fn exported, imported ->
         assert exported["os_version"] == imported["os_version"]
@@ -843,15 +845,112 @@ defmodule Plausible.Imported.CSVImporterTest do
         assert_in_delta exported["visit_duration"], imported["visit_duration"], 1
       end)
 
-      # NOTE: os version breakdown's visitors difference is up to almost 80%
+      # NOTE: os version breakdown's visitors difference is up to almost 34%
-      assert summary(field(exported_os_versions, "visitors")) == [1, 1, 3, 10.75, 165]
+      assert summary(field(exported_os_versions, "visitors")) == [1, 1.0, 2.0, 7.0, 380]
-      assert summary(field(imported_os_versions, "visitors")) == [1, 1.75, 4.5, 14.5, 258]
+      assert summary(field(imported_os_versions, "visitors")) == [1, 1.0, 2.0, 7.0, 393]
 
       assert summary(
                pairwise(exported_os_versions, imported_os_versions, fn exported, imported ->
                  abs(1 - exported["visitors"] / imported["visitors"])
                end)
-             ) == [0, 0, 0.16985645933014354, 0.3401162790697675, 0.75]
+             ) == [0.0, 0.0, 0.0, 0.0, 0.33333333333333337]
+
+      # goals
+      exported_goals =
+        breakdown.(exported_site, "event:goal", metrics: "visitors,events,conversion_rate")
+
+      imported_goals =
+        breakdown.(imported_site, "event:goal", metrics: "visitors,events,conversion_rate")
+
+      # NOTE: goal breakdown's visitors difference is up to 20%
+      assert summary(field(exported_goals, "visitors")) == [1, 6.5, 12.0, 41.0, 70]
+      assert summary(field(imported_goals, "visitors")) == [1, 6.5, 12.0, 50.0, 88]
+
+      assert summary(
+               pairwise(exported_goals, imported_goals, fn exported, imported ->
+                 abs(1 - exported["visitors"] / imported["visitors"])
+               end)
+             ) == [0.0, 0.0, 0.0, 0.10227272727272729, 0.20454545454545459]
+
+      pairwise(exported_goals, imported_goals, fn exported, imported ->
+        assert exported["events"] == imported["events"]
+      end)
+
+      # NOTE: goal breakdown's conversion rate difference is up to 18%
+      assert summary(field(exported_goals, "conversion_rate")) == [
+               0.1,
+               0.55,
+               1.0,
+               3.3,
+               5.6
+             ]
+
+      assert summary(field(imported_goals, "conversion_rate")) == [
+               0.1,
+               0.5,
+               0.9,
+               3.8499999999999996,
+               6.8
+             ]
+
+      assert summary(
+               pairwise(exported_goals, imported_goals, fn exported, imported ->
+                 abs(1 - exported["conversion_rate"] / imported["conversion_rate"])
+               end)
+             ) == [
+               0.0,
+               0.05555555555555558,
+               0.11111111111111116,
+               0.14379084967320266,
+               0.17647058823529416
+             ]
+
+      # url property breakdown
+      exported_url_props =
+        breakdown.(exported_site, "event:props:url",
+          filters: "event:goal==Outbound Link: Click",
+          metrics: "visitors,events,conversion_rate"
+        )
+
+      imported_url_props =
+        breakdown.(imported_site, "event:props:url",
+          filters: "event:goal==Outbound Link: Click",
+          metrics: "visitors,events,conversion_rate"
+        )
+
+      pairwise(exported_url_props, imported_url_props, fn exported, imported ->
+        assert exported["visitors"] == imported["visitors"]
+        assert exported["events"] == imported["events"]
+      end)
+
+      # NOTE: url property breakdown's conversion rate difference is up to 20%
+      assert summary(field(exported_url_props, "conversion_rate")) == [0.1, 0.1, 0.1, 0.1, 0.8]
+      assert summary(field(imported_url_props, "conversion_rate")) == [0.1, 0.1, 0.1, 0.1, 0.8]
+
+      assert summary(
+               pairwise(exported_url_props, imported_url_props, fn exported, imported ->
+                 abs(1 - exported["conversion_rate"] / imported["conversion_rate"])
+               end)
+             ) == [0.0, 0.0, 0.0, 0.0, 0.19999999999999996]
+
+      # path property breakdown
+      exported_path_props =
+        breakdown.(exported_site, "event:props:path",
+          filters: "event:goal==404",
+          metrics: "visitors,events,conversion_rate"
+        )
+
+      imported_path_props =
+        breakdown.(imported_site, "event:props:path",
+          filters: "event:goal==404",
+          metrics: "visitors,events,conversion_rate"
+        )
+
+      pairwise(exported_path_props, imported_path_props, fn exported, imported ->
+        assert exported["visitors"] == imported["visitors"]
+        assert exported["events"] == imported["events"]
+        assert exported["conversion_rate"] == imported["conversion_rate"]
+      end)
     end
   end
 

test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs

@@ -3164,7 +3164,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
       assert %{"source" => "Google", "events" => 1} = breakdown_and_first.("visit:source")
     end
 
-    for goal_name <- ["Outbound Link: Click", "File Download"] do
+    for goal_name <- Plausible.Imported.goals_with_url() do
       test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do
         insert(:goal, event_name: unquote(goal_name), site: site)
         site_import = insert(:site_import, site: site)
@@ -3221,5 +3221,63 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
                ]
       end
     end
+
+    for goal_name <- Plausible.Imported.goals_with_path() do
+      test "returns path breakdown for #{goal_name} goal", %{conn: conn, site: site} do
+        insert(:goal, event_name: unquote(goal_name), site: site)
+        site_import = insert(:site_import, site: site)
+
+        populate_stats(site, site_import.id, [
+          build(:event,
+            name: unquote(goal_name),
+            "meta.key": ["path"],
+            "meta.value": ["/one"]
+          ),
+          build(:imported_custom_events,
+            name: unquote(goal_name),
+            visitors: 2,
+            events: 5,
+            path: "/one"
+          ),
+          build(:imported_custom_events,
+            name: unquote(goal_name),
+            visitors: 5,
+            events: 10,
+            path: "/two"
+          ),
+          build(:imported_custom_events,
+            name: "some goal",
+            visitors: 5,
+            events: 10
+          ),
+          build(:imported_visitors, visitors: 9)
+        ])
+
+        conn =
+          get(conn, "/api/v1/stats/breakdown", %{
+            "site_id" => site.domain,
+            "period" => "day",
+            "property" => "event:props:path",
+            "filters" => "event:goal==#{unquote(goal_name)}",
+            "metrics" => "visitors,events,conversion_rate",
+            "with_imported" => "true"
+          })
+
+        assert json_response(conn, 200)["results"] == [
+                 %{
+                   "visitors" => 5,
+                   "path" => "/two",
+                   "events" => 10,
+                   "conversion_rate" => 50.0
+                 },
+                 %{
+                   "visitors" => 3,
+                   "path" => "/one",
+                   "events" => 6,
+                   "conversion_rate" => 30.0
+                 }
+               ]
+      end
+    end
   end
 end