mirror
/
analytics
mirror of https://github.com/plausible/analytics.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Import acquisiton channel from GA4 (#4814) 

* Import acquisiton channel from GA4

* Remove unnecessary step

* Fix spelling

* Remove migration

* Show empty channel in imported data as (not set)

* Revert "Remove migration"

This reverts commit da0b9403e4.

* Fix channel suggestions with imported data

* Merge group field entries

* Add note about channel mappings

* Revert "Revert "Remove migration""

This reverts commit 7958a46c5c.
This commit is contained in:
Uku Taht 2024-11-14 20:22:14 +02:00 committed by GitHub
parent 738db2df98
commit b0933e1730
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 848 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

635
fixture/ga4_report_imported_sources.json
View File

File diff suppressed because it is too large Load Diff

1
lib/plausible/google/ga4/report_request.ex
View File

@ -51,6 +51,7 @@ defmodule Plausible.Google.GA4.ReportRequest do
dimensions: [
"date",
"sessionSource",
"sessionDefaultChannelGroup",
"sessionMedium",
"sessionCampaignName",
"sessionManualAdContent",

2
lib/plausible/imported/google_analytics4.ex
View File

@ -175,6 +175,8 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
import_id: import_id,
date: get_date(row),
source: row.dimensions |> Map.fetch!("sessionSource") |> parse_source(),
# GA4 channels map 1-1 to Plausible channels
channel: row.dimensions |> Map.fetch!("sessionDefaultChannelGroup"),
referrer: nil,
# Only `source` exists in GA4 API
utm_source: nil,

1
lib/plausible/imported/source.ex
View File

@ -8,6 +8,7 @@ defmodule Plausible.Imported.Source do
field :import_id, Ch, type: "UInt64"
field :date, :date
field :source, :string
field :channel, Ch, type: "LowCardinality(String)"
field :referrer, :string
field :utm_source, :string
field :utm_medium, :string

1
lib/plausible/stats/imported/base.ex
View File

@ -12,6 +12,7 @@ defmodule Plausible.Stats.Imported.Base do
@property_to_table_mappings %{
"visit:source" => "imported_sources",
"visit:channel" => "imported_sources",
"visit:referrer" => "imported_sources",
"visit:utm_source" => "imported_sources",
"visit:utm_medium" => "imported_sources",

1
lib/plausible/stats/imported/imported.ex
View File

@ -197,6 +197,7 @@ defmodule Plausible.Stats.Imported do
@filter_suggestions_mapping %{
referrer_source: :source,
acquisition_channel: :channel,
screen_size: :device,
pathname: :page
}

2
lib/plausible/stats/imported/sql/expression.ex
View File

@ -236,7 +236,7 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
end
defp select_group_fields(q, dimension, key, _query)
when dimension in ["visit:device", "visit:browser"] do
when dimension in ["visit:device", "visit:browser", "visit:channel"] do
select_merge_as(q, [i], %{
key =>
fragment(

185
test/plausible_web/controllers/api/stats_controller/imported_test.exs
View File

@ -164,7 +164,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "organic",
"sessionSource" => "duckduckgo.com"
"sessionSource" => "duckduckgo.com",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "0",
@ -181,7 +182,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210131",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "organic",
"sessionSource" => "google.com"
"sessionSource" => "google.com",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "1",
@ -198,7 +200,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "paid",
"sessionSource" => "google.com"
"sessionSource" => "google.com",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "1",
@ -215,7 +218,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "social",
"sessionSource" => "Twitter"
"sessionSource" => "Twitter",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "1",
@ -232,7 +236,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210131",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "email",
"sessionSource" => "A Nice Newsletter"
"sessionSource" => "A Nice Newsletter",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "1",
@ -249,7 +254,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "(none)",
"sessionSource" => "(direct)"
"sessionSource" => "(direct)",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "1",
@ -283,6 +289,140 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
]
end
test "Channels are imported", %{conn: conn, site: site, import_id: import_id} do
populate_stats(site, [
# Organic Search
build(:pageview,
referrer_source: "Bing",
timestamp: ~N[2021-01-01 00:00:00]
),
# Paid Search
build(:pageview,
referrer_source: "Google",
utm_medium: "paid",
timestamp: ~N[2021-01-01 00:00:00]
),
# Direct
build(:pageview,
timestamp: ~N[2021-01-01 00:00:00]
)
])
import_data(
[
%{
dimensions: %{
"sessionManualAdContent" => "",
"sessionCampaignName" => "",
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "organic",
"sessionSource" => "duckduckgo.com",
"sessionDefaultChannelGroup" => "Organic Search"
},
metrics: %{
"bounces" => "0",
"userEngagementDuration" => "60",
"sessions" => "1",
"totalUsers" => "1",
"screenPageViews" => "1"
}
},
%{
dimensions: %{
"sessionManualAdContent" => "",
"sessionCampaignName" => "",
"date" => "20210131",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "organic",
"sessionSource" => "google.com",
"sessionDefaultChannelGroup" => "Organic Search"
},
metrics: %{
"bounces" => "1",
"userEngagementDuration" => "60",
"sessions" => "1",
"totalUsers" => "1",
"screenPageViews" => "1"
}
},
%{
dimensions: %{
"sessionManualAdContent" => "",
"sessionCampaignName" => "",
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "paid",
"sessionSource" => "google.com",
"sessionDefaultChannelGroup" => "Paid Search"
},
metrics: %{
"bounces" => "1",
"userEngagementDuration" => "60",
"sessions" => "1",
"totalUsers" => "1",
"screenPageViews" => "1"
}
},
%{
dimensions: %{
"sessionManualAdContent" => "",
"sessionCampaignName" => "",
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "(none)",
"sessionSource" => "(direct)",
"sessionDefaultChannelGroup" => "Direct"
},
metrics: %{
"bounces" => "1",
"userEngagementDuration" => "60",
"sessions" => "1",
"totalUsers" => "1",
"screenPageViews" => "1"
}
},
%{
dimensions: %{
"sessionManualAdContent" => "",
"sessionCampaignName" => "",
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "(none)",
"sessionSource" => "(direct)",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "1",
"userEngagementDuration" => "60",
"sessions" => "1",
"totalUsers" => "1",
"screenPageViews" => "1"
}
}
],
site.id,
import_id,
"imported_sources"
)
results =
conn
|> get(
"/api/stats/#{site.domain}/channels?period=month&date=2021-01-01&with_imported=true"
)
|> json_response(200)
|> Map.get("results")
|> Enum.sort()
assert results == [
%{"name" => "(not set)", "visitors" => 1},
%{"name" => "Direct", "visitors" => 2},
%{"name" => "Organic Search", "visitors" => 3},
%{"name" => "Paid Search", "visitors" => 2}
]
end
test "UTM mediums data imported from Google Analytics", %{
conn: conn,
site: site,
@ -308,7 +448,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "social",
"sessionSource" => "Twitter"
"sessionSource" => "Twitter",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "1",
@ -325,7 +466,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "(none)",
"sessionSource" => "(direct)"
"sessionSource" => "(direct)",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "1",
@ -376,7 +518,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "social",
"sessionSource" => "Twitter"
"sessionSource" => "Twitter",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "1",
@ -393,7 +536,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "email",
"sessionSource" => "Gmail"
"sessionSource" => "Gmail",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "0",
@ -410,7 +554,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "email",
"sessionSource" => "Gmail"
"sessionSource" => "Gmail",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "0",
@ -468,7 +613,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "oat milk",
"sessionMedium" => "paid",
"sessionSource" => "Google"
"sessionSource" => "Google",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "1",
@ -485,7 +631,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "Sweden",
"sessionMedium" => "paid",
"sessionSource" => "Google"
"sessionSource" => "Google",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "0",
@ -502,7 +649,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "(not set)",
"sessionMedium" => "paid",
"sessionSource" => "Google"
"sessionSource" => "Google",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "0",
@ -559,7 +707,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "paid",
"sessionSource" => "Google"
"sessionSource" => "Google",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "1",
@ -576,7 +725,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "paid",
"sessionSource" => "Google"
"sessionSource" => "Google",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "0",
@ -593,7 +743,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
"date" => "20210101",
"sessionGoogleAdsKeyword" => "",
"sessionMedium" => "paid",
"sessionSource" => "Google"
"sessionSource" => "Google",
"sessionDefaultChannelGroup" => ""
},
metrics: %{
"bounces" => "0",

25
test/plausible_web/controllers/api/stats_controller/suggestions_test.exs
View File

@ -1142,6 +1142,31 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
%{"value" => "Bing", "label" => "Bing"}
]
end
test "merges channel suggestions from native and imported data #{label}", %{
conn: conn,
site: site,
site_import: site_import
} do
populate_stats(site, site_import.id, [
build(:pageview, timestamp: ~N[2019-01-01 23:00:01], referrer_source: "Bing"),
build(:pageview, timestamp: ~N[2019-01-01 23:30:01], referrer_source: "Bing"),
build(:pageview, timestamp: ~N[2019-01-01 23:40:01], referrer_source: "Bing"),
build(:pageview, timestamp: ~N[2019-01-01 23:00:01], referrer_source: "Google"),
build(:imported_sources, date: ~D[2019-01-01], channel: "Organic Social", pageviews: 3)
])
conn =
get(
conn,
"/api/stats/#{site.domain}/suggestions/channel?period=month&date=2019-01-01&q=#{unquote(q)}&with_imported=true"
)
assert json_response(conn, 200) == [
%{"value" => "Organic Search", "label" => "Organic Search"},
%{"value" => "Organic Social", "label" => "Organic Social"}
]
end
end
for {q, label} <- [{"", "without filter"}, {"o", "with filter"}] do

3
test/plausible_web/controllers/stats_controller_test.exs
View File

@ -398,6 +398,7 @@ defmodule PlausibleWeb.StatsControllerTest do
build(:imported_pages, page: "/test", pageviews: 1),
build(:imported_sources,
source: "Google",
channel: "Paid Search",
utm_medium: "search",
utm_campaign: "ads",
utm_source: "google",
@ -473,10 +474,10 @@ defmodule PlausibleWeb.StatsControllerTest do
[""]
]
# Dummy - imported data is not actually included in exported CSVs yet
{~c"channels.csv", data} ->
assert parse_csv(data) == [
["name", "visitors", "bounce_rate", "visit_duration"],
["Paid Search", "1", "0.0", "10.0"],
[""]
]

16
test/support/factory.ex
View File

@ -136,14 +136,18 @@ defmodule Plausible.Factory do
}
end
def pageview_factory do
Map.put(event_factory(), :name, "pageview")
def pageview_factory(attrs) do
Map.put(event_factory(attrs), :name, "pageview")
end
def event_factory do
def event_factory(attrs) do
if Map.get(attrs, :acquisition_channel) do
raise "Acquisition channel cannot be written directly since it's a materialized column."
end
hostname = sequence(:domain, &"example-#{&1}.com")
%Plausible.ClickhouseEventV2{
event = %Plausible.ClickhouseEventV2{
hostname: hostname,
site_id: Enum.random(1000..10_000),
pathname: "/",
@ -151,6 +155,10 @@ defmodule Plausible.Factory do
user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()),
session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate())
}
event
|> merge_attributes(attrs)
|> evaluate_lazy_attributes()
end
def goal_factory(attrs) do