Import acquisiton channel from GA4 (#4814)

* Import acquisiton channel from GA4

* Remove unnecessary step

* Fix spelling

* Remove migration

* Show empty channel in imported data as (not set)

* Revert "Remove migration"

This reverts commit da0b9403e4.

* Fix channel suggestions with imported data

* Merge group field entries

* Add note about channel mappings

* Revert "Revert "Remove migration""

This reverts commit 7958a46c5c.

lib/plausible/google/ga4/report_request.ex

@@ -51,6 +51,7 @@ defmodule Plausible.Google.GA4.ReportRequest do
         dimensions: [
           "date",
           "sessionSource",
+          "sessionDefaultChannelGroup",
           "sessionMedium",
           "sessionCampaignName",
           "sessionManualAdContent",

lib/plausible/imported/google_analytics4.ex

@@ -175,6 +175,8 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
       import_id: import_id,
       date: get_date(row),
       source: row.dimensions |> Map.fetch!("sessionSource") |> parse_source(),
+      # GA4 channels map 1-1 to Plausible channels
+      channel: row.dimensions |> Map.fetch!("sessionDefaultChannelGroup"),
       referrer: nil,
       # Only `source` exists in GA4 API
       utm_source: nil,

lib/plausible/imported/source.ex

@@ -8,6 +8,7 @@ defmodule Plausible.Imported.Source do
     field :import_id, Ch, type: "UInt64"
     field :date, :date
     field :source, :string
+    field :channel, Ch, type: "LowCardinality(String)"
     field :referrer, :string
     field :utm_source, :string
     field :utm_medium, :string

lib/plausible/stats/imported/base.ex

@@ -12,6 +12,7 @@ defmodule Plausible.Stats.Imported.Base do
 
   @property_to_table_mappings %{
     "visit:source" => "imported_sources",
+    "visit:channel" => "imported_sources",
     "visit:referrer" => "imported_sources",
     "visit:utm_source" => "imported_sources",
     "visit:utm_medium" => "imported_sources",

lib/plausible/stats/imported/imported.ex

@@ -197,6 +197,7 @@ defmodule Plausible.Stats.Imported do
 
   @filter_suggestions_mapping %{
     referrer_source: :source,
+    acquisition_channel: :channel,
     screen_size: :device,
     pathname: :page
   }

lib/plausible/stats/imported/sql/expression.ex

@@ -236,7 +236,7 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
   end
 
   defp select_group_fields(q, dimension, key, _query)
-       when dimension in ["visit:device", "visit:browser"] do
+       when dimension in ["visit:device", "visit:browser", "visit:channel"] do
     select_merge_as(q, [i], %{
       key =>
         fragment(

test/plausible_web/controllers/api/stats_controller/imported_test.exs

@@ -164,7 +164,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "",
                 "sessionMedium" => "organic",
-                "sessionSource" => "duckduckgo.com"
+                "sessionSource" => "duckduckgo.com",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "0",
@@ -181,7 +182,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210131",
                 "sessionGoogleAdsKeyword" => "",
                 "sessionMedium" => "organic",
-                "sessionSource" => "google.com"
+                "sessionSource" => "google.com",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "1",
@@ -198,7 +200,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "",
                 "sessionMedium" => "paid",
-                "sessionSource" => "google.com"
+                "sessionSource" => "google.com",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "1",
@@ -215,7 +218,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "",
                 "sessionMedium" => "social",
-                "sessionSource" => "Twitter"
+                "sessionSource" => "Twitter",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "1",
@@ -232,7 +236,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210131",
                 "sessionGoogleAdsKeyword" => "",
                 "sessionMedium" => "email",
-                "sessionSource" => "A Nice Newsletter"
+                "sessionSource" => "A Nice Newsletter",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "1",
@@ -249,7 +254,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "",
                 "sessionMedium" => "(none)",
-                "sessionSource" => "(direct)"
+                "sessionSource" => "(direct)",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "1",
@@ -283,6 +289,140 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                ]
       end
 
+      test "Channels are imported", %{conn: conn, site: site, import_id: import_id} do
+        populate_stats(site, [
+          # Organic Search
+          build(:pageview,
+            referrer_source: "Bing",
+            timestamp: ~N[2021-01-01 00:00:00]
+          ),
+          # Paid Search
+          build(:pageview,
+            referrer_source: "Google",
+            utm_medium: "paid",
+            timestamp: ~N[2021-01-01 00:00:00]
+          ),
+          # Direct
+          build(:pageview,
+            timestamp: ~N[2021-01-01 00:00:00]
+          )
+        ])
+
+        import_data(
+          [
+            %{
+              dimensions: %{
+                "sessionManualAdContent" => "",
+                "sessionCampaignName" => "",
+                "date" => "20210101",
+                "sessionGoogleAdsKeyword" => "",
+                "sessionMedium" => "organic",
+                "sessionSource" => "duckduckgo.com",
+                "sessionDefaultChannelGroup" => "Organic Search"
+              },
+              metrics: %{
+                "bounces" => "0",
+                "userEngagementDuration" => "60",
+                "sessions" => "1",
+                "totalUsers" => "1",
+                "screenPageViews" => "1"
+              }
+            },
+            %{
+              dimensions: %{
+                "sessionManualAdContent" => "",
+                "sessionCampaignName" => "",
+                "date" => "20210131",
+                "sessionGoogleAdsKeyword" => "",
+                "sessionMedium" => "organic",
+                "sessionSource" => "google.com",
+                "sessionDefaultChannelGroup" => "Organic Search"
+              },
+              metrics: %{
+                "bounces" => "1",
+                "userEngagementDuration" => "60",
+                "sessions" => "1",
+                "totalUsers" => "1",
+                "screenPageViews" => "1"
+              }
+            },
+            %{
+              dimensions: %{
+                "sessionManualAdContent" => "",
+                "sessionCampaignName" => "",
+                "date" => "20210101",
+                "sessionGoogleAdsKeyword" => "",
+                "sessionMedium" => "paid",
+                "sessionSource" => "google.com",
+                "sessionDefaultChannelGroup" => "Paid Search"
+              },
+              metrics: %{
+                "bounces" => "1",
+                "userEngagementDuration" => "60",
+                "sessions" => "1",
+                "totalUsers" => "1",
+                "screenPageViews" => "1"
+              }
+            },
+            %{
+              dimensions: %{
+                "sessionManualAdContent" => "",
+                "sessionCampaignName" => "",
+                "date" => "20210101",
+                "sessionGoogleAdsKeyword" => "",
+                "sessionMedium" => "(none)",
+                "sessionSource" => "(direct)",
+                "sessionDefaultChannelGroup" => "Direct"
+              },
+              metrics: %{
+                "bounces" => "1",
+                "userEngagementDuration" => "60",
+                "sessions" => "1",
+                "totalUsers" => "1",
+                "screenPageViews" => "1"
+              }
+            },
+            %{
+              dimensions: %{
+                "sessionManualAdContent" => "",
+                "sessionCampaignName" => "",
+                "date" => "20210101",
+                "sessionGoogleAdsKeyword" => "",
+                "sessionMedium" => "(none)",
+                "sessionSource" => "(direct)",
+                "sessionDefaultChannelGroup" => ""
+              },
+              metrics: %{
+                "bounces" => "1",
+                "userEngagementDuration" => "60",
+                "sessions" => "1",
+                "totalUsers" => "1",
+                "screenPageViews" => "1"
+              }
+            }
+          ],
+          site.id,
+          import_id,
+          "imported_sources"
+        )
+
+        results =
+          conn
+          |> get(
+            "/api/stats/#{site.domain}/channels?period=month&date=2021-01-01&with_imported=true"
+          )
+          |> json_response(200)
+          |> Map.get("results")
+          |> Enum.sort()
+
+        assert results == [
+                 %{"name" => "(not set)", "visitors" => 1},
+                 %{"name" => "Direct", "visitors" => 2},
+                 %{"name" => "Organic Search", "visitors" => 3},
+                 %{"name" => "Paid Search", "visitors" => 2}
+               ]
+      end
+
       test "UTM mediums data imported from Google Analytics", %{
         conn: conn,
         site: site,
@@ -308,7 +448,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "",
                 "sessionMedium" => "social",
-                "sessionSource" => "Twitter"
+                "sessionSource" => "Twitter",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "1",
@@ -325,7 +466,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "",
                 "sessionMedium" => "(none)",
-                "sessionSource" => "(direct)"
+                "sessionSource" => "(direct)",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "1",
@@ -376,7 +518,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "",
                 "sessionMedium" => "social",
-                "sessionSource" => "Twitter"
+                "sessionSource" => "Twitter",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "1",
@@ -393,7 +536,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "",
                 "sessionMedium" => "email",
-                "sessionSource" => "Gmail"
+                "sessionSource" => "Gmail",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "0",
@@ -410,7 +554,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "",
                 "sessionMedium" => "email",
-                "sessionSource" => "Gmail"
+                "sessionSource" => "Gmail",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "0",
@@ -468,7 +613,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "oat milk",
                 "sessionMedium" => "paid",
-                "sessionSource" => "Google"
+                "sessionSource" => "Google",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "1",
@@ -485,7 +631,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "Sweden",
                 "sessionMedium" => "paid",
-                "sessionSource" => "Google"
+                "sessionSource" => "Google",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "0",
@@ -502,7 +649,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "(not set)",
                 "sessionMedium" => "paid",
-                "sessionSource" => "Google"
+                "sessionSource" => "Google",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "0",
@@ -559,7 +707,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "",
                 "sessionMedium" => "paid",
-                "sessionSource" => "Google"
+                "sessionSource" => "Google",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "1",
@@ -576,7 +725,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "",
                 "sessionMedium" => "paid",
-                "sessionSource" => "Google"
+                "sessionSource" => "Google",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "0",
@@ -593,7 +743,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
                 "date" => "20210101",
                 "sessionGoogleAdsKeyword" => "",
                 "sessionMedium" => "paid",
-                "sessionSource" => "Google"
+                "sessionSource" => "Google",
+                "sessionDefaultChannelGroup" => ""
               },
               metrics: %{
                 "bounces" => "0",

test/plausible_web/controllers/api/stats_controller/suggestions_test.exs

@@ -1142,6 +1142,31 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
                  %{"value" => "Bing", "label" => "Bing"}
                ]
       end
+
+      test "merges channel suggestions from native and imported data #{label}", %{
+        conn: conn,
+        site: site,
+        site_import: site_import
+      } do
+        populate_stats(site, site_import.id, [
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], referrer_source: "Bing"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:30:01], referrer_source: "Bing"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:40:01], referrer_source: "Bing"),
+          build(:pageview, timestamp: ~N[2019-01-01 23:00:01], referrer_source: "Google"),
+          build(:imported_sources, date: ~D[2019-01-01], channel: "Organic Social", pageviews: 3)
+        ])
+
+        conn =
+          get(
+            conn,
+            "/api/stats/#{site.domain}/suggestions/channel?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"
+          )
+
+        assert json_response(conn, 200) == [
+                 %{"value" => "Organic Search", "label" => "Organic Search"},
+                 %{"value" => "Organic Social", "label" => "Organic Social"}
+               ]
+      end
     end
 
     for {q, label} <- [{"", "without filter"}, {"o", "with filter"}] do

test/plausible_web/controllers/stats_controller_test.exs

@@ -398,6 +398,7 @@ defmodule PlausibleWeb.StatsControllerTest do
         build(:imported_pages, page: "/test", pageviews: 1),
         build(:imported_sources,
           source: "Google",
+          channel: "Paid Search",
           utm_medium: "search",
           utm_campaign: "ads",
           utm_source: "google",
@@ -473,10 +474,10 @@ defmodule PlausibleWeb.StatsControllerTest do
                    [""]
                  ]
 
-        # Dummy - imported data is not actually included in exported CSVs yet
         {~c"channels.csv", data} ->
           assert parse_csv(data) == [
                    ["name", "visitors", "bounce_rate", "visit_duration"],
+                   ["Paid Search", "1", "0.0", "10.0"],
                    [""]
                  ]
 

test/support/factory.ex

@@ -136,14 +136,18 @@ defmodule Plausible.Factory do
     }
   end
 
-  def pageview_factory do
+  def pageview_factory(attrs) do
-    Map.put(event_factory(), :name, "pageview")
+    Map.put(event_factory(attrs), :name, "pageview")
+  end
+
+  def event_factory(attrs) do
+    if Map.get(attrs, :acquisition_channel) do
+      raise "Acquisition channel cannot be written directly since it's a materialized column."
     end
 
-  def event_factory do
     hostname = sequence(:domain, &"example-#{&1}.com")
 
-    %Plausible.ClickhouseEventV2{
+    event = %Plausible.ClickhouseEventV2{
       hostname: hostname,
       site_id: Enum.random(1000..10_000),
       pathname: "/",
@@ -151,6 +155,10 @@ defmodule Plausible.Factory do
       user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
       session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
     }
+
+    event
+    |> merge_attributes(attrs)
+    |> evaluate_lazy_attributes()
   end
 
   def goal_factory(attrs) do