Add custom props to full export (#5666)

* Add custom props to full export

* Pass full `site` struct to `export_queries`

* Export only internal props if plan lacks custom props

* Add changelog entry

* Add spot check test for custom props

* Do not generate cartesian product of prop/value pairs 🤦

CHANGELOG.md

@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.
 - Custom events can now be marked as non-interactive in events API and tracker script. Events marked as non-interactive are not counted towards bounce rate.
 - Ability to leave team via Team Settings > Leave Team
 - Stats APIv2 now supports `include.trim_relative_date_range`. This option allows trimming empty values after current time for `day`, `month` and `year` date_range values.
+- Properties are now included in full site exports done via Site Settings > Imports & Exports
 
 ### Removed
 

lib/plausible/exports.ex

@@ -208,13 +208,13 @@ defmodule Plausible.Exports do
   Builds Ecto queries to export data from `events_v2` and `sessions_v2`
   tables into the format of `imported_*` tables for a website.
   """
-  @spec export_queries(pos_integer,
+  @spec export_queries(Plausible.Site.t(),
           extname: String.t(),
           date_range: Date.Range.t(),
           timezone: String.t()
         ) ::
           %{String.t() => Ecto.Query.t()}
-  def export_queries(site_id, opts \\ []) do
+  def export_queries(site, opts \\ []) do
     extname = opts[:extname] || ".csv"
     date_range = opts[:date_range]
     timezone = opts[:timezone] || "UTC"
@@ -231,18 +231,18 @@ defmodule Plausible.Exports do
     filename = fn name -> name <> suffix end
 
     %{
-      filename.("imported_visitors") => export_visitors_q(site_id, timezone, date_range),
+      filename.("imported_visitors") => export_visitors_q(site, timezone, date_range),
-      filename.("imported_sources") => export_sources_q(site_id, timezone, date_range),
+      filename.("imported_sources") => export_sources_q(site, timezone, date_range),
-      filename.("imported_pages") => export_pages_q(site_id, timezone, date_range),
+      filename.("imported_pages") => export_pages_q(site, timezone, date_range),
-      filename.("imported_entry_pages") => export_entry_pages_q(site_id, timezone, date_range),
+      filename.("imported_entry_pages") => export_entry_pages_q(site, timezone, date_range),
-      filename.("imported_exit_pages") => export_exit_pages_q(site_id, timezone, date_range),
+      filename.("imported_exit_pages") => export_exit_pages_q(site, timezone, date_range),
-      filename.("imported_custom_events") =>
+      filename.("imported_custom_events") => export_custom_events_q(site, timezone, date_range),
-        export_custom_events_q(site_id, timezone, date_range),
+      filename.("imported_locations") => export_locations_q(site, timezone, date_range),
-      filename.("imported_locations") => export_locations_q(site_id, timezone, date_range),
+      filename.("imported_devices") => export_devices_q(site, timezone, date_range),
-      filename.("imported_devices") => export_devices_q(site_id, timezone, date_range),
+      filename.("imported_browsers") => export_browsers_q(site, timezone, date_range),
-      filename.("imported_browsers") => export_browsers_q(site_id, timezone, date_range),
       filename.("imported_operating_systems") =>
-        export_operating_systems_q(site_id, timezone, date_range)
+        export_operating_systems_q(site, timezone, date_range),
+      filename.("imported_custom_props") => export_custom_props_q(site, timezone, date_range)
     }
   end
 
@@ -336,10 +336,10 @@ defmodule Plausible.Exports do
     end
   end
 
-  defp export_visitors_q(site_id, timezone, date_range) do
+  defp export_visitors_q(site, timezone, date_range) do
     visitors_sessions_q =
       from s in sampled("sessions_v2"),
-        where: ^export_filter(site_id, date_range),
+        where: ^export_filter(site.id, date_range),
         group_by: selected_as(:date),
         select: %{
           date: date(s.timestamp, ^timezone),
@@ -351,7 +351,7 @@ defmodule Plausible.Exports do
 
     visitors_events_q =
       from e in sampled("events_v2"),
-        where: ^export_filter(site_id, date_range),
+        where: ^export_filter(site.id, date_range),
         group_by: selected_as(:date),
         select: %{
           date: date(e.timestamp, ^timezone),
@@ -381,9 +381,9 @@ defmodule Plausible.Exports do
       ]
   end
 
-  defp export_sources_q(site_id, timezone, date_range) do
+  defp export_sources_q(site, timezone, date_range) do
     from s in sampled("sessions_v2"),
-      where: ^export_filter(site_id, date_range),
+      where: ^export_filter(site.id, date_range),
       group_by: [
         selected_as(:date),
         selected_as(:source),
@@ -412,10 +412,10 @@ defmodule Plausible.Exports do
       ]
   end
 
-  defp export_pages_q(site_id, timezone, date_range) do
+  defp export_pages_q(site, timezone, date_range) do
     base_q =
       from(e in sampled("events_v2"),
-        where: ^export_filter(site_id, date_range),
+        where: ^export_filter(site.id, date_range),
         where: [name: "pageview"],
         group_by: [selected_as(:date), selected_as(:page)],
         order_by: selected_as(:date)
@@ -423,7 +423,7 @@ defmodule Plausible.Exports do
 
     max_scroll_depth_per_session_q =
       from(e in "events_v2",
-        where: ^export_filter(site_id, date_range),
+        where: ^export_filter(site.id, date_range),
         where: e.name == "engagement" and e.scroll_depth <= 100,
         select: %{
           date: date(e.timestamp, ^timezone),
@@ -465,7 +465,7 @@ defmodule Plausible.Exports do
           selected_as(fragment("any(?)", s.total_scroll_depth_visits), :total_scroll_depth_visits)
       }
     )
-    |> add_time_on_page_columns(site_id, timezone, date_range)
+    |> add_time_on_page_columns(site.id, timezone, date_range)
   end
 
   defp add_time_on_page_columns(q, site_id, timezone, date_range) do
@@ -508,9 +508,9 @@ defmodule Plausible.Exports do
     end
   end
 
-  defp export_entry_pages_q(site_id, timezone, date_range) do
+  defp export_entry_pages_q(site, timezone, date_range) do
     from s in sampled("sessions_v2"),
-      where: ^export_filter(site_id, date_range),
+      where: ^export_filter(site.id, date_range),
       group_by: [selected_as(:date), s.entry_page],
       order_by: selected_as(:date),
       select: [
@@ -527,9 +527,9 @@ defmodule Plausible.Exports do
       ]
   end
 
-  defp export_exit_pages_q(site_id, timezone, date_range) do
+  defp export_exit_pages_q(site, timezone, date_range) do
     from s in sampled("sessions_v2"),
-      where: ^export_filter(site_id, date_range),
+      where: ^export_filter(site.id, date_range),
       group_by: [selected_as(:date), s.exit_page],
       order_by: selected_as(:date),
       select: [
@@ -546,9 +546,9 @@ defmodule Plausible.Exports do
       ]
   end
 
-  defp export_custom_events_q(site_id, timezone, date_range) do
+  defp export_custom_events_q(site, timezone, date_range) do
     from e in sampled("events_v2"),
-      where: ^export_filter(site_id, date_range),
+      where: ^export_filter(site.id, date_range),
       where: e.name != "pageview",
       group_by: [
         selected_as(:date),
@@ -583,9 +583,37 @@ defmodule Plausible.Exports do
       ]
   end
 
-  defp export_locations_q(site_id, timezone, date_range) do
+  defp export_custom_props_q(site, timezone, date_range) do
+    query =
+      from e in sampled("events_v2"),
+        join: pv in fragment("arrayZip(`meta.key`, `meta.value`)"),
+        on: true,
+        hints: "ARRAY",
+        where: ^export_filter(site.id, date_range),
+        group_by: [
+          selected_as(:date),
+          selected_as(:property),
+          selected_as(:value)
+        ],
+        order_by: selected_as(:date),
+        select: [
+          date(e.timestamp, ^timezone),
+          selected_as(fragment("tupleElement(?, 1)", pv), :property),
+          selected_as(fragment("tupleElement(?, 2)", pv), :value),
+          visitors(e),
+          selected_as(scale_sample(fragment("count()")), :events)
+        ]
+
+    if Plausible.Billing.Feature.Props.enabled?(site) do
+      query
+    else
+      where(query, [], selected_as(:property) in ^Plausible.Props.internal_keys())
+    end
+  end
+
+  defp export_locations_q(site, timezone, date_range) do
     from s in sampled("sessions_v2"),
-      where: ^export_filter(site_id, date_range),
+      where: ^export_filter(site.id, date_range),
       where: s.country_code != "\0\0" and s.country_code != "ZZ",
       group_by: [selected_as(:date), s.country_code, s.subdivision1_code, s.city_geoname_id],
       order_by: selected_as(:date),
@@ -602,9 +630,9 @@ defmodule Plausible.Exports do
       ]
   end
 
-  defp export_devices_q(site_id, timezone, date_range) do
+  defp export_devices_q(site, timezone, date_range) do
     from s in sampled("sessions_v2"),
-      where: ^export_filter(site_id, date_range),
+      where: ^export_filter(site.id, date_range),
       group_by: [selected_as(:date), s.screen_size],
       order_by: selected_as(:date),
       select: [
@@ -618,9 +646,9 @@ defmodule Plausible.Exports do
       ]
   end
 
-  defp export_browsers_q(site_id, timezone, date_range) do
+  defp export_browsers_q(site, timezone, date_range) do
     from s in sampled("sessions_v2"),
-      where: ^export_filter(site_id, date_range),
+      where: ^export_filter(site.id, date_range),
       group_by: [selected_as(:date), s.browser, s.browser_version],
       order_by: selected_as(:date),
       select: [
@@ -635,9 +663,9 @@ defmodule Plausible.Exports do
       ]
   end
 
-  defp export_operating_systems_q(site_id, timezone, date_range) do
+  defp export_operating_systems_q(site, timezone, date_range) do
     from s in sampled("sessions_v2"),
-      where: ^export_filter(site_id, date_range),
+      where: ^export_filter(site.id, date_range),
       group_by: [selected_as(:date), s.operating_system, s.operating_system_version],
       order_by: selected_as(:date),
       select: [
@@ -661,7 +689,7 @@ defmodule Plausible.Exports do
 
       DBConnection.run(pool, fn conn ->
         conn
-        |> stream_archive(export_queries(_site_id = 1), format: "CSVWithNames")
+        |> stream_archive(export_queries(site), format: "CSVWithNames")
         |> Stream.into(File.stream!("export.zip"))
         |> Stream.run()
       end)

lib/plausible/imported/csv_importer.ex

@@ -18,6 +18,8 @@ defmodule Plausible.Imported.CSVImporter do
 
   @impl true
   def parse_args(%{"uploads" => uploads, "storage" => storage}) do
+    uploads = Enum.reject(uploads, &String.starts_with?(&1["filename"], "imported_custom_props_"))
+
     [uploads: uploads, storage: storage]
   end
 
@@ -208,6 +210,14 @@ defmodule Plausible.Imported.CSVImporter do
   def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil)
   def date_range([]), do: nil
 
+  defp date_range(
+         [%{"filename" => "imported_custom_props_" <> _} | uploads],
+         prev_start_date,
+         prev_end_date
+       ) do
+    date_range(uploads, prev_start_date, prev_end_date)
+  end
+
   defp date_range([upload | uploads], prev_start_date, prev_end_date) do
     filename =
       case upload do

lib/workers/export_analytics.ex

@@ -33,7 +33,7 @@ defmodule Plausible.Workers.ExportAnalytics do
     %Date.Range{} = date_range = Exports.date_range(site.id, site.timezone)
 
     queries =
-      Exports.export_queries(site_id,
+      Exports.export_queries(site,
         date_range: date_range,
         timezone: site.timezone,
         extname: ".csv"

test/plausible/exports_test.exs

@@ -9,12 +9,13 @@ defmodule Plausible.ExportsTest do
     setup [:create_user, :create_site]
 
     test "returns named ecto queries", %{site: site} do
-      queries = Plausible.Exports.export_queries(site.id)
+      queries = Plausible.Exports.export_queries(site)
       assert queries |> Map.values() |> Enum.all?(&match?(%Ecto.Query{}, &1))
 
       assert Map.keys(queries) == [
                "imported_browsers.csv",
                "imported_custom_events.csv",
+               "imported_custom_props.csv",
                "imported_devices.csv",
                "imported_entry_pages.csv",
                "imported_exit_pages.csv",
@@ -28,13 +29,14 @@ defmodule Plausible.ExportsTest do
 
     test "with date range", %{site: site} do
       queries =
-        Plausible.Exports.export_queries(site.id,
+        Plausible.Exports.export_queries(site,
           date_range: Date.range(~D[2023-01-01], ~D[2024-03-12])
         )
 
       assert Map.keys(queries) == [
                "imported_browsers_20230101_20240312.csv",
                "imported_custom_events_20230101_20240312.csv",
+               "imported_custom_props_20230101_20240312.csv",
                "imported_devices_20230101_20240312.csv",
                "imported_entry_pages_20230101_20240312.csv",
                "imported_exit_pages_20230101_20240312.csv",
@@ -47,11 +49,12 @@ defmodule Plausible.ExportsTest do
     end
 
     test "with custom extension", %{site: site} do
-      queries = Plausible.Exports.export_queries(site.id, extname: ".ch")
+      queries = Plausible.Exports.export_queries(site, extname: ".ch")
 
       assert Map.keys(queries) == [
                "imported_browsers.ch",
                "imported_custom_events.ch",
+               "imported_custom_props.ch",
                "imported_devices.ch",
                "imported_entry_pages.ch",
                "imported_exit_pages.ch",

test/plausible/imported/csv_importer_test.exs

@@ -608,7 +608,7 @@ defmodule Plausible.Imported.CSVImporterTest do
         imported_site: imported_site
       }
 
-      %{site_import: site_import} =
+      %{site_import: site_import, exported_files: exported_files} =
         initial_context
         |> export_archive()
         |> assert_email_notification()
@@ -617,6 +617,16 @@ defmodule Plausible.Imported.CSVImporterTest do
         |> upload_csvs()
         |> run_import()
 
+      assert custom_props_export =
+               exported_files
+               |> Enum.find(&String.contains?(&1, "imported_custom_props_"))
+               |> File.read!()
+               |> String.split("\n")
+
+      assert ~s|"date","property","value","visitors","events"| in custom_props_export
+      assert ~s|"2024-04-01","author","Marko Saric",43,57| in custom_props_export
+      assert ~s|"2024-04-01","category","Posts",43,56| in custom_props_export
+
       assert %SiteImport{
                start_date: ~D[2024-04-01],
                end_date: ~D[2024-04-30],