mirror
/
analytics
mirror of https://github.com/plausible/analytics.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Populate new columns in imports and exports (#3969) 

* Extend `Imported*` schemas with newly added columns

* Populate newly added `Imported*` fields in GA4 imports

* Extend exports with newly added fields

* Extend CSV importer to ingest new fields

* Fix alias shadowing error

* Add more extensive GA4 import fixtures

* Apply rounding and casting to sampled visits
This commit is contained in:
Adrian Gruntkowski 2024-04-04 10:33:19 +02:00 committed by GitHub
parent bd73fc8266
commit e6d83e946f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 331746 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6193
fixture/ga4_report_imported_browsers.json
View File

File diff suppressed because it is too large Load Diff

2435
fixture/ga4_report_imported_devices.json
View File

File diff suppressed because it is too large Load Diff

79168
fixture/ga4_report_imported_entry_pages.json
View File

File diff suppressed because it is too large Load Diff

75519
fixture/ga4_report_imported_locations.json
View File

File diff suppressed because it is too large Load Diff

31991
fixture/ga4_report_imported_operating_systems.json
View File

File diff suppressed because it is too large Load Diff

90040
fixture/ga4_report_imported_pages.json
View File

File diff suppressed because it is too large Load Diff

45695
fixture/ga4_report_imported_sources.json
View File

File diff suppressed because it is too large Load Diff

712
fixture/ga4_report_imported_visitors.json
View File

@ -10,7 +10,7 @@
"kind": "analyticsData#runReport", "kind": "analyticsData#runReport",
"metadata": { "metadata": {
"currencyCode": "USD", "currencyCode": "USD",
"timeZone": "Europe/Warsaw" "timeZone": "Europe/Amsterdam"
}, },
"metricHeaders": [ "metricHeaders": [
{ {
@ -34,77 +34,749 @@
"type": "TYPE_SECONDS" "type": "TYPE_SECONDS"
} }
], ],
"rowCount": 3, "rowCount": 31,
"rows": [ "rows": [
{ {
"dimensionValues": [ "dimensionValues": [
{ {
"value": "20240226" "value": "20240131"
} }
], ],
"metricValues": [ "metricValues": [
{ {
"value": "1" "value": "141"
}, },
{ {
"value": "1" "value": "195"
}, },
{ {
"value": "1" "value": "56"
}, },
{ {
"value": "1" "value": "146"
}, },
{ {
"value": "0" "value": "4815"
} }
] ]
}, },
{ {
"dimensionValues": [ "dimensionValues": [
{ {
"value": "20240224" "value": "20240130"
} }
], ],
"metricValues": [ "metricValues": [
{ {
"value": "2" "value": "115"
}, },
{ {
"value": "2" "value": "171"
}, },
{ {
"value": "2" "value": "42"
}, },
{ {
"value": "2" "value": "115"
}, },
{ {
"value": "0" "value": "3481"
} }
] ]
}, },
{ {
"dimensionValues": [ "dimensionValues": [
{ {
"value": "20240222" "value": "20240129"
} }
], ],
"metricValues": [ "metricValues": [
{ {
"value": "3" "value": "105"
}, },
{ {
"value": "13" "value": "153"
}, },
{ {
"value": "0" "value": "54"
}, },
{ {
"value": "4" "value": "113"
}, },
{ {
"value": "29" "value": "2895"
}
]
},
{
"dimensionValues": [
{
"value": "20240128"
}
],
"metricValues": [
{
"value": "184"
},
{
"value": "259"
},
{
"value": "74"
},
{
"value": "190"
},
{
"value": "6861"
}
]
},
{
"dimensionValues": [
{
"value": "20240127"
}
],
"metricValues": [
{
"value": "155"
},
{
"value": "202"
},
{
"value": "67"
},
{
"value": "158"
},
{
"value": "5259"
}
]
},
{
"dimensionValues": [
{
"value": "20240126"
}
],
"metricValues": [
{
"value": "132"
},
{
"value": "189"
},
{
"value": "57"
},
{
"value": "142"
},
{
"value": "5545"
}
]
},
{
"dimensionValues": [
{
"value": "20240125"
}
],
"metricValues": [
{
"value": "151"
},
{
"value": "229"
},
{
"value": "71"
},
{
"value": "155"
},
{
"value": "5365"
}
]
},
{
"dimensionValues": [
{
"value": "20240124"
}
],
"metricValues": [
{
"value": "137"
},
{
"value": "221"
},
{
"value": "64"
},
{
"value": "148"
},
{
"value": "4454"
}
]
},
{
"dimensionValues": [
{
"value": "20240123"
}
],
"metricValues": [
{
"value": "166"
},
{
"value": "267"
},
{
"value": "61"
},
{
"value": "171"
},
{
"value": "4850"
}
]
},
{
"dimensionValues": [
{
"value": "20240122"
}
],
"metricValues": [
{
"value": "161"
},
{
"value": "233"
},
{
"value": "73"
},
{
"value": "165"
},
{
"value": "4871"
}
]
},
{
"dimensionValues": [
{
"value": "20240121"
}
],
"metricValues": [
{
"value": "254"
},
{
"value": "397"
},
{
"value": "114"
},
{
"value": "267"
},
{
"value": "8845"
}
]
},
{
"dimensionValues": [
{
"value": "20240120"
}
],
"metricValues": [
{
"value": "214"
},
{
"value": "383"
},
{
"value": "100"
},
{
"value": "223"
},
{
"value": "6914"
}
]
},
{
"dimensionValues": [
{
"value": "20240119"
}
],
"metricValues": [
{
"value": "381"
},
{
"value": "5571"
},
{
"value": "100"
},
{
"value": "228"
},
{
"value": "98549"
}
]
},
{
"dimensionValues": [
{
"value": "20240118"
}
],
"metricValues": [
{
"value": "177"
},
{
"value": "278"
},
{
"value": "86"
},
{
"value": "186"
},
{
"value": "5553"
}
]
},
{
"dimensionValues": [
{
"value": "20240117"
}
],
"metricValues": [
{
"value": "204"
},
{
"value": "271"
},
{
"value": "97"
},
{
"value": "216"
},
{
"value": "6777"
}
]
},
{
"dimensionValues": [
{
"value": "20240116"
}
],
"metricValues": [
{
"value": "200"
},
{
"value": "280"
},
{
"value": "65"
},
{
"value": "203"
},
{
"value": "9381"
}
]
},
{
"dimensionValues": [
{
"value": "20240115"
}
],
"metricValues": [
{
"value": "255"
},
{
"value": "311"
},
{
"value": "86"
},
{
"value": "269"
},
{
"value": "12365"
}
]
},
{
"dimensionValues": [
{
"value": "20240114"
}
],
"metricValues": [
{
"value": "243"
},
{
"value": "302"
},
{
"value": "86"
},
{
"value": "257"
},
{
"value": "11073"
}
]
},
{
"dimensionValues": [
{
"value": "20240113"
}
],
"metricValues": [
{
"value": "231"
},
{
"value": "276"
},
{
"value": "92"
},
{
"value": "244"
},
{
"value": "10084"
}
]
},
{
"dimensionValues": [
{
"value": "20240112"
}
],
"metricValues": [
{
"value": "215"
},
{
"value": "265"
},
{
"value": "75"
},
{
"value": "225"
},
{
"value": "10195"
}
]
},
{
"dimensionValues": [
{
"value": "20240111"
}
],
"metricValues": [
{
"value": "183"
},
{
"value": "235"
},
{
"value": "64"
},
{
"value": "191"
},
{
"value": "8200"
}
]
},
{
"dimensionValues": [
{
"value": "20240110"
}
],
"metricValues": [
{
"value": "190"
},
{
"value": "244"
},
{
"value": "61"
},
{
"value": "203"
},
{
"value": "9056"
}
]
},
{
"dimensionValues": [
{
"value": "20240109"
}
],
"metricValues": [
{
"value": "201"
},
{
"value": "290"
},
{
"value": "63"
},
{
"value": "213"
},
{
"value": "11651"
}
]
},
{
"dimensionValues": [
{
"value": "20240108"
}
],
"metricValues": [
{
"value": "195"
},
{
"value": "235"
},
{
"value": "71"
},
{
"value": "209"
},
{
"value": "7946"
}
]
},
{
"dimensionValues": [
{
"value": "20240107"
}
],
"metricValues": [
{
"value": "205"
},
{
"value": "274"
},
{
"value": "80"
},
{
"value": "218"
},
{
"value": "10048"
}
]
},
{
"dimensionValues": [
{
"value": "20240106"
}
],
"metricValues": [
{
"value": "256"
},
{
"value": "303"
},
{
"value": "119"
},
{
"value": "281"
},
{
"value": "8440"
}
]
},
{
"dimensionValues": [
{
"value": "20240105"
}
],
"metricValues": [
{
"value": "226"
},
{
"value": "270"
},
{
"value": "83"
},
{
"value": "237"
},
{
"value": "9311"
}
]
},
{
"dimensionValues": [
{
"value": "20240104"
}
],
"metricValues": [
{
"value": "223"
},
{
"value": "274"
},
{
"value": "81"
},
{
"value": "236"
},
{
"value": "12051"
}
]
},
{
"dimensionValues": [
{
"value": "20240103"
}
],
"metricValues": [
{
"value": "233"
},
{
"value": "283"
},
{
"value": "79"
},
{
"value": "245"
},
{
"value": "10628"
}
]
},
{
"dimensionValues": [
{
"value": "20240102"
}
],
"metricValues": [
{
"value": "203"
},
{
"value": "269"
},
{
"value": "60"
},
{
"value": "210"
},
{
"value": "11168"
}
]
},
{
"dimensionValues": [
{
"value": "20240101"
}
],
"metricValues": [
{
"value": "191"
},
{
"value": "224"
},
{
"value": "71"
},
{
"value": "197"
},
{
"value": "8080"
} }
] ]
} }

96
lib/plausible/exports.ex
View File

@ -118,51 +118,27 @@ defmodule Plausible.Exports do
end end
end end
defmacrop pageviews(t) do
quote do
selected_as(
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).pageviews),
:pageviews
)
end
end
@spec export_visitors_q(pos_integer) :: Ecto.Query.t() @spec export_visitors_q(pos_integer) :: Ecto.Query.t()
def export_visitors_q(site_id) do def export_visitors_q(site_id) do
visitors_sessions_q = from s in sampled("sessions_v2"),
from s in sampled("sessions_v2"), where: s.site_id == ^site_id,
where: s.site_id == ^site_id, group_by: selected_as(:date),
group_by: selected_as(:date),
select: %{
date: date(s.start),
bounces: bounces(s),
visits: visits(s),
visit_duration: visit_duration(s)
# NOTE: can we use just sessions_v2 table in this query? sum(pageviews) and visitors(s)?
# visitors: visitors(s)
}
visitors_events_q =
from e in sampled("events_v2"),
where: e.site_id == ^site_id,
group_by: selected_as(:date),
select: %{
date: date(e.timestamp),
visitors: visitors(e),
pageviews:
selected_as(
fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),
:pageviews
)
}
visitors_q =
"e"
|> with_cte("e", as: ^visitors_events_q)
|> with_cte("s", as: ^visitors_sessions_q)
from e in visitors_q,
full_join: s in "s",
on: e.date == s.date,
order_by: selected_as(:date),
select: [ select: [
selected_as(fragment("greatest(?,?)", s.date, e.date), :date), date(s.start),
e.visitors, visitors(s),
e.pageviews, pageviews(s),
s.bounces, bounces(s),
s.visits, visits(s),
s.visit_duration visit_duration(s)
] ]
end end
@ -173,6 +149,8 @@ defmodule Plausible.Exports do
group_by: [ group_by: [
selected_as(:date), selected_as(:date),
selected_as(:source), selected_as(:source),
s.referrer,
s.utm_source,
s.utm_medium, s.utm_medium,
s.utm_campaign, s.utm_campaign,
s.utm_content, s.utm_content,
@ -182,10 +160,13 @@ defmodule Plausible.Exports do
select: [ select: [
date(s.start), date(s.start),
selected_as(s.referrer_source, :source), selected_as(s.referrer_source, :source),
s.referrer,
s.utm_source,
s.utm_medium, s.utm_medium,
s.utm_campaign, s.utm_campaign,
s.utm_content, s.utm_content,
s.utm_term, s.utm_term,
pageviews(s),
visitors(s), visitors(s),
visits(s), visits(s),
visit_duration(s), visit_duration(s),
@ -210,6 +191,7 @@ defmodule Plausible.Exports do
hostname: e.hostname, hostname: e.hostname,
name: e.name, name: e.name,
user_id: e.user_id, user_id: e.user_id,
session_id: e.session_id,
_sample_factor: fragment("_sample_factor") _sample_factor: fragment("_sample_factor")
} }
@ -220,6 +202,10 @@ defmodule Plausible.Exports do
date(e.timestamp), date(e.timestamp),
selected_as(fragment("any(?)", e.hostname), :hostname), selected_as(fragment("any(?)", e.hostname), :hostname),
selected_as(e.pathname, :page), selected_as(e.pathname, :page),
selected_as(
fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", e.session_id),
:visits
),
visitors(e), visitors(e),
selected_as( selected_as(
fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name), fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),
@ -252,7 +238,8 @@ defmodule Plausible.Exports do
:entrances :entrances
), ),
visit_duration(s), visit_duration(s),
bounces(s) bounces(s),
pageviews(s)
] ]
end end
@ -266,10 +253,13 @@ defmodule Plausible.Exports do
date(s.start), date(s.start),
s.exit_page, s.exit_page,
visitors(s), visitors(s),
visit_duration(s),
selected_as( selected_as(
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign), fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
:exits :exits
) ),
bounces(s),
pageviews(s)
] ]
end end
@ -288,7 +278,8 @@ defmodule Plausible.Exports do
visitors(s), visitors(s),
visits(s), visits(s),
visit_duration(s), visit_duration(s),
bounces(s) bounces(s),
pageviews(s)
] ]
end end
@ -304,7 +295,8 @@ defmodule Plausible.Exports do
visitors(s), visitors(s),
visits(s), visits(s),
visit_duration(s), visit_duration(s),
bounces(s) bounces(s),
pageviews(s)
] ]
end end
@ -312,15 +304,17 @@ defmodule Plausible.Exports do
def export_browsers_q(site_id) do def export_browsers_q(site_id) do
from s in sampled("sessions_v2"), from s in sampled("sessions_v2"),
where: s.site_id == ^site_id, where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.browser], group_by: [selected_as(:date), s.browser, s.browser_version],
order_by: selected_as(:date), order_by: selected_as(:date),
select: [ select: [
date(s.start), date(s.start),
s.browser, s.browser,
s.browser_version,
visitors(s), visitors(s),
visits(s), visits(s),
visit_duration(s), visit_duration(s),
bounces(s) bounces(s),
pageviews(s)
] ]
end end
@ -328,15 +322,17 @@ defmodule Plausible.Exports do
def export_operating_systems_q(site_id) do def export_operating_systems_q(site_id) do
from s in sampled("sessions_v2"), from s in sampled("sessions_v2"),
where: s.site_id == ^site_id, where: s.site_id == ^site_id,
group_by: [selected_as(:date), s.operating_system], group_by: [selected_as(:date), s.operating_system, s.operating_system_version],
order_by: selected_as(:date), order_by: selected_as(:date),
select: [ select: [
date(s.start), date(s.start),
s.operating_system, s.operating_system,
s.operating_system_version,
visitors(s), visitors(s),
visits(s), visits(s),
visit_duration(s), visit_duration(s),
bounces(s) bounces(s),
pageviews(s)
] ]
end end

21
lib/plausible/google/ga4/report_request.ex
View File

@ -46,9 +46,11 @@ defmodule Plausible.Google.GA4.ReportRequest do
"sessionMedium", "sessionMedium",
"sessionCampaignName", "sessionCampaignName",
"sessionManualAdContent", "sessionManualAdContent",
"sessionGoogleAdsKeyword" "sessionGoogleAdsKeyword",
"pageReferrer"
], ],
metrics: [ metrics: [
"screenPageViews",
"totalUsers", "totalUsers",
"sessions", "sessions",
"bounces = sessions - engagedSessions", "bounces = sessions - engagedSessions",
@ -59,12 +61,13 @@ defmodule Plausible.Google.GA4.ReportRequest do
dataset: "imported_pages", dataset: "imported_pages",
dimensions: ["date", "hostName", "pagePath"], dimensions: ["date", "hostName", "pagePath"],
# NOTE: no exits as GA4 DATA API does not provide that metric # NOTE: no exits as GA4 DATA API does not provide that metric
metrics: ["totalUsers", "screenPageViews", "userEngagementDuration"] metrics: ["totalUsers", "screenPageViews", "sessions", "userEngagementDuration"]
}, },
%__MODULE__{ %__MODULE__{
dataset: "imported_entry_pages", dataset: "imported_entry_pages",
dimensions: ["date", "landingPage"], dimensions: ["date", "landingPage"],
metrics: [ metrics: [
"screenPageViews",
"totalUsers", "totalUsers",
"sessions", "sessions",
"userEngagementDuration", "userEngagementDuration",
@ -75,12 +78,19 @@ defmodule Plausible.Google.GA4.ReportRequest do
# %__MODULE__{ # %__MODULE__{
# dataset: "imported_exit_pages", # dataset: "imported_exit_pages",
# dimensions: ["date", "ga:exitPagePath"], # dimensions: ["date", "ga:exitPagePath"],
# metrics: ["totalUsers", "sessions"] # metrics: [
# "totalUsers",
# "sessions",
# "screenPageViews",
# "userEngagementDuration",
# "bounces = sessions - engagedSessions"
# ]
# }, # },
%__MODULE__{ %__MODULE__{
dataset: "imported_locations", dataset: "imported_locations",
dimensions: ["date", "countryId", "region", "city"], dimensions: ["date", "countryId", "region", "city"],
metrics: [ metrics: [
"screenPageViews",
"totalUsers", "totalUsers",
"sessions", "sessions",
"bounces = sessions - engagedSessions", "bounces = sessions - engagedSessions",
@ -91,6 +101,7 @@ defmodule Plausible.Google.GA4.ReportRequest do
dataset: "imported_devices", dataset: "imported_devices",
dimensions: ["date", "deviceCategory"], dimensions: ["date", "deviceCategory"],
metrics: [ metrics: [
"screenPageViews",
"totalUsers", "totalUsers",
"sessions", "sessions",
"bounces = sessions - engagedSessions", "bounces = sessions - engagedSessions",
@ -101,6 +112,7 @@ defmodule Plausible.Google.GA4.ReportRequest do
dataset: "imported_browsers", dataset: "imported_browsers",
dimensions: ["date", "browser"], dimensions: ["date", "browser"],
metrics: [ metrics: [
"screenPageViews",
"totalUsers", "totalUsers",
"sessions", "sessions",
"bounces = sessions - engagedSessions", "bounces = sessions - engagedSessions",
@ -109,8 +121,9 @@ defmodule Plausible.Google.GA4.ReportRequest do
}, },
%__MODULE__{ %__MODULE__{
dataset: "imported_operating_systems", dataset: "imported_operating_systems",
dimensions: ["date", "operatingSystem"], dimensions: ["date", "operatingSystem", "operatingSystemVersion"],
metrics: [ metrics: [
"screenPageViews",
"totalUsers", "totalUsers",
"sessions", "sessions",
"bounces = sessions - engagedSessions", "bounces = sessions - engagedSessions",

2
lib/plausible/imported/browser.ex
View File

@ -8,9 +8,11 @@ defmodule Plausible.Imported.Browser do
field :import_id, Ch, type: "UInt64" field :import_id, Ch, type: "UInt64"
field :date, :date field :date, :date
field :browser, :string field :browser, :string
field :browser_version, :string
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

17
lib/plausible/imported/csv_importer.ex
View File

@ -83,20 +83,21 @@ defmodule Plausible.Imported.CSVImporter do
input_structures = %{ input_structures = %{
"imported_browsers" => "imported_browsers" =>
"date Date, browser String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32", "date Date, browser String, browser_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
"imported_devices" => "imported_devices" =>
"date Date, device String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32", "date Date, device String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
"imported_entry_pages" => "imported_entry_pages" =>
"date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32", "date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
"imported_exit_pages" => "date Date, exit_page String, visitors UInt64, exits UInt64", "imported_exit_pages" =>
"date Date, exit_page String, visitors UInt64, visit_duration UInt64, exits UInt64, bounces UInt32, pageviews UInt64",
"imported_locations" => "imported_locations" =>
"date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32", "date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
"imported_operating_systems" => "imported_operating_systems" =>
"date Date, operating_system String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32", "date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
"imported_pages" => "imported_pages" =>
"date Date, hostname String, page String, visitors UInt64, pageviews UInt64, exits UInt64, time_on_page UInt64", "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, exits UInt64, time_on_page UInt64",
"imported_sources" => "imported_sources" =>
"date Date, source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32", "date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_visitors" => "imported_visitors" =>
"date Date, visitors UInt64, pageviews UInt64, bounces UInt64, visits UInt64, visit_duration UInt64" "date Date, visitors UInt64, pageviews UInt64, bounces UInt64, visits UInt64, visit_duration UInt64"
} }

1
lib/plausible/imported/device.ex
View File

@ -11,6 +11,7 @@ defmodule Plausible.Imported.Device do
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

1
lib/plausible/imported/entry_page.ex
View File

@ -11,6 +11,7 @@ defmodule Plausible.Imported.EntryPage do
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :entrances, Ch, type: "UInt64" field :entrances, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

5
lib/plausible/imported/exit_page.ex
View File

@ -8,7 +8,10 @@ defmodule Plausible.Imported.ExitPage do
field :import_id, Ch, type: "UInt64" field :import_id, Ch, type: "UInt64"
field :date, :date field :date, :date
field :exit_page, :string field :exit_page, :string
field :visitors, Ch, type: "UInt64"
field :exits, Ch, type: "UInt64" field :exits, Ch, type: "UInt64"
field :visitors, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32"
end end
end end

25
lib/plausible/imported/google_analytics4.ex
View File

@ -96,17 +96,28 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
end end
defp new_from_report(site_id, import_id, "imported_sources", row) do defp new_from_report(site_id, import_id, "imported_sources", row) do
referrer_uri = row.dimensions |> Map.fetch!("pageReferrer") |> URI.parse()
referrer =
if PlausibleWeb.RefInspector.right_uri?(referrer_uri) do
PlausibleWeb.RefInspector.format_referrer(referrer_uri)
end
%{ %{
site_id: site_id, site_id: site_id,
import_id: import_id, import_id: import_id,
date: get_date(row), date: get_date(row),
source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(), source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(),
referrer: referrer,
# Only `source` exists in GA4 API
utm_source: nil,
utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(), utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(),
utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(), utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(),
utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(), utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(),
utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(), utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
} }
@ -120,6 +131,7 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""), hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""),
page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path), page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
# NOTE: no exits metric in GA4 API currently # NOTE: no exits metric in GA4 API currently
exits: 0, exits: 0,
@ -136,6 +148,7 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(), entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(), visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number() bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
} }
end end
@ -148,7 +161,10 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
# date: get_date(row), # date: get_date(row),
# exit_page: Map.fetch!(row.dimensions, "exitPage"), # exit_page: Map.fetch!(row.dimensions, "exitPage"),
# visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), # visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
# exits: row.metrics |> Map.fetch!("sessions") |> parse_number() # exits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
# visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
# pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
# bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
# } # }
# end # end
@ -166,6 +182,7 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
city: city_data && city_data.id, city: city_data && city_data.id,
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
} }
@ -179,6 +196,7 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(), device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
} }
@ -202,8 +220,11 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
import_id: import_id, import_id: import_id,
date: get_date(row), date: get_date(row),
browser: Map.get(@browser_google_to_plausible, browser, browser), browser: Map.get(@browser_google_to_plausible, browser, browser),
# Does not exist in GA4 API
browser_version: nil,
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
} }
@ -223,8 +244,10 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
import_id: import_id, import_id: import_id,
date: get_date(row), date: get_date(row),
operating_system: Map.get(@os_google_to_plausible, os, os), operating_system: Map.get(@os_google_to_plausible, os, os),
operating_system_version: row.dimensions |> Map.fetch!("operatingSystemVersion"),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
} }

1
lib/plausible/imported/location.ex
View File

@ -13,6 +13,7 @@ defmodule Plausible.Imported.Location do
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

2
lib/plausible/imported/operating_system.ex
View File

@ -8,9 +8,11 @@ defmodule Plausible.Imported.OperatingSystem do
field :import_id, Ch, type: "UInt64" field :import_id, Ch, type: "UInt64"
field :date, :date field :date, :date
field :operating_system, :string field :operating_system, :string
field :operating_system_version, :string
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

1
lib/plausible/imported/page.ex
View File

@ -9,6 +9,7 @@ defmodule Plausible.Imported.Page do
field :date, :date field :date, :date
field :hostname, :string field :hostname, :string
field :page, :string field :page, :string
field :visits, Ch, type: "UInt64"
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64" field :pageviews, Ch, type: "UInt64"
field :exits, Ch, type: "UInt64" field :exits, Ch, type: "UInt64"

3
lib/plausible/imported/source.ex
View File

@ -8,6 +8,8 @@ defmodule Plausible.Imported.Source do
field :import_id, Ch, type: "UInt64" field :import_id, Ch, type: "UInt64"
field :date, :date field :date, :date
field :source, :string field :source, :string
field :referrer, :string
field :utm_source, :string
field :utm_medium, :string field :utm_medium, :string
field :utm_campaign, :string field :utm_campaign, :string
field :utm_content, :string field :utm_content, :string
@ -15,6 +17,7 @@ defmodule Plausible.Imported.Source do
field :visitors, Ch, type: "UInt64" field :visitors, Ch, type: "UInt64"
field :visits, Ch, type: "UInt64" field :visits, Ch, type: "UInt64"
field :visit_duration, Ch, type: "UInt64" field :visit_duration, Ch, type: "UInt64"
field :pageviews, Ch, type: "UInt64"
field :bounces, Ch, type: "UInt32" field :bounces, Ch, type: "UInt32"
end end
end end

16
test/plausible/imported/google_analytics4_test.exs
View File

@ -82,15 +82,15 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
Enum.each(Plausible.Imported.tables(), fn table -> Enum.each(Plausible.Imported.tables(), fn table ->
count = count =
case table do case table do
"imported_sources" -> 3 "imported_sources" -> 1090
"imported_visitors" -> 3 "imported_visitors" -> 31
"imported_pages" -> 8 "imported_pages" -> 3340
"imported_entry_pages" -> 4 "imported_entry_pages" -> 2934
"imported_exit_pages" -> 0 "imported_exit_pages" -> 0
"imported_locations" -> 4 "imported_locations" -> 2291
"imported_devices" -> 4 "imported_devices" -> 93
"imported_browsers" -> 5 "imported_browsers" -> 233
"imported_operating_systems" -> 4 "imported_operating_systems" -> 1068
end end
query = from(imported in table, where: imported.site_id == ^site.id) query = from(imported in table, where: imported.site_id == ^site.id)