mirror
/
analytics
mirror of https://github.com/plausible/analytics.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implement basics of GA4 import (#3851) 

* Implement LV date input using flatpickr

* Implement basics of GA4 import (very dirty WIP)

* Split Google HTTP API into UA and GA4 specific parts

* Add a quick way to record GA4 API responses

* Add first GA4 import fixtures with GA4 Data API responses

* Extract GA4 and UA specific logic form Google API

* Extract UA and GA4 specific actions to distinct controllers

* Add integration test for GA4 importer

* Update GA4 fixtures

* Test GA4 API

* Add debug logging and fix paginating through API results in in GA4 import

* Revert "Implement LV date input using flatpickr"

This reverts commit c696f8ee39d5702f27015c09a4f079ca124cc7bb.

* Fix note
This commit is contained in:
Adrian Gruntkowski 2024-03-12 18:08:25 +01:00 committed by GitHub
parent f2350b5165
commit 4d7d88cfec
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
45 changed files with 3118 additions and 598 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/config.exs
View File

@ -45,7 +45,7 @@ config :ref_inspector,
config :plausible, config :plausible,
paddle_api: Plausible.Billing.PaddleApi, paddle_api: Plausible.Billing.PaddleApi,
google_api: Plausible.Google.Api google_api: Plausible.Google.API
config :plausible, config :plausible,
# 30 minutes # 30 minutes

2
config/test.exs
View File

@ -16,7 +16,7 @@ config :plausible, Plausible.Mailer, adapter: Bamboo.TestAdapter
config :plausible, config :plausible,
paddle_api: Plausible.PaddleApi.Mock, paddle_api: Plausible.PaddleApi.Mock,
google_api: Plausible.Google.Api.Mock google_api: Plausible.Google.API.Mock
config :bamboo, :refute_timeout, 10 config :bamboo, :refute_timeout, 10

41
fixture/ga4_list_properties.json Normal file
View File

@ -0,0 +1,41 @@
{
"accountSummaries": [
{
"account": "accounts/28425178",
"displayName": "account.one",
"name": "accountSummaries/28425178",
"propertySummaries": [
{
"displayName": "account.one - GA4",
"parent": "accounts/28425178",
"property": "properties/428685906",
"propertyType": "PROPERTY_TYPE_ORDINARY"
}
]
},
{
"account": "accounts/45336102",
"displayName": "account.two",
"name": "accountSummaries/45336102"
},
{
"account": "accounts/54516992",
"displayName": "Demo Account",
"name": "accountSummaries/54516992",
"propertySummaries": [
{
"displayName": "GA4 - Flood-It!",
"parent": "accounts/54516992",
"property": "properties/153293282",
"propertyType": "PROPERTY_TYPE_ORDINARY"
},
{
"displayName": "GA4 - Google Merch Shop",
"parent": "accounts/54516992",
"property": "properties/213025502",
"propertyType": "PROPERTY_TYPE_ORDINARY"
}
]
}
]
}

161
fixture/ga4_report_imported_browsers.json Normal file
View File

@ -0,0 +1,161 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
},
{
"name": "browser"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "totalUsers",
"type": "TYPE_INTEGER"
},
{
"name": "sessions",
"type": "TYPE_INTEGER"
},
{
"name": "bounces",
"type": "TYPE_INTEGER"
},
{
"name": "userEngagementDuration",
"type": "TYPE_SECONDS"
}
],
"rowCount": 5,
"rows": [
{
"dimensionValues": [
{
"value": "20240226"
},
{
"value": "Safari"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240224"
},
{
"value": "Chrome"
}
],
"metricValues": [
{
"value": "2"
},
{
"value": "2"
},
{
"value": "2"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "Chrome"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
},
{
"value": "4"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "Firefox"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "2"
},
{
"value": "0"
},
{
"value": "21"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "Safari"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
},
{
"value": "4"
}
]
}
]
}
]
}

137
fixture/ga4_report_imported_devices.json Normal file
View File

@ -0,0 +1,137 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
},
{
"name": "deviceCategory"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "totalUsers",
"type": "TYPE_INTEGER"
},
{
"name": "sessions",
"type": "TYPE_INTEGER"
},
{
"name": "bounces",
"type": "TYPE_INTEGER"
},
{
"name": "userEngagementDuration",
"type": "TYPE_SECONDS"
}
],
"rowCount": 4,
"rows": [
{
"dimensionValues": [
{
"value": "20240226"
},
{
"value": "mobile"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240224"
},
{
"value": "desktop"
}
],
"metricValues": [
{
"value": "2"
},
{
"value": "2"
},
{
"value": "2"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "desktop"
}
],
"metricValues": [
{
"value": "2"
},
{
"value": "3"
},
{
"value": "0"
},
{
"value": "25"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "mobile"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
},
{
"value": "4"
}
]
}
]
}
]
}

137
fixture/ga4_report_imported_entry_pages.json Normal file
View File

@ -0,0 +1,137 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
},
{
"name": "landingPage"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "totalUsers",
"type": "TYPE_INTEGER"
},
{
"name": "sessions",
"type": "TYPE_INTEGER"
},
{
"name": "userEngagementDuration",
"type": "TYPE_SECONDS"
},
{
"name": "bounces",
"type": "TYPE_INTEGER"
}
],
"rowCount": 4,
"rows": [
{
"dimensionValues": [
{
"value": "20240226"
},
{
"value": "/blog/firstpost"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
},
{
"value": "1"
}
]
},
{
"dimensionValues": [
{
"value": "20240224"
},
{
"value": "/"
}
],
"metricValues": [
{
"value": "2"
},
{
"value": "2"
},
{
"value": "0"
},
{
"value": "2"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "/"
}
],
"metricValues": [
{
"value": "2"
},
{
"value": "3"
},
{
"value": "25"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "/blog/unicode-in-elixir"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "4"
},
{
"value": "0"
}
]
}
]
}
]
}

167
fixture/ga4_report_imported_locations.json Normal file
View File

@ -0,0 +1,167 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
},
{
"name": "countryId"
},
{
"name": "region"
},
{
"name": "city"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "totalUsers",
"type": "TYPE_INTEGER"
},
{
"name": "sessions",
"type": "TYPE_INTEGER"
},
{
"name": "bounces",
"type": "TYPE_INTEGER"
},
{
"name": "userEngagementDuration",
"type": "TYPE_SECONDS"
}
],
"rowCount": 4,
"rows": [
{
"dimensionValues": [
{
"value": "20240226"
},
{
"value": "PL"
},
{
"value": "Masovian Voivodeship"
},
{
"value": "Warsaw"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240224"
},
{
"value": "US"
},
{
"value": "California"
},
{
"value": "(not set)"
}
],
"metricValues": [
{
"value": "2"
},
{
"value": "2"
},
{
"value": "2"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "PL"
},
{
"value": "Pomeranian Voivodeship"
},
{
"value": "Gdansk"
}
],
"metricValues": [
{
"value": "2"
},
{
"value": "3"
},
{
"value": "0"
},
{
"value": "25"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "ES"
},
{
"value": "Catalonia"
},
{
"value": "Barcelona"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
},
{
"value": "4"
}
]
}
]
}
]
}

137
fixture/ga4_report_imported_operating_systems.json Normal file
View File

@ -0,0 +1,137 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
},
{
"name": "operatingSystem"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "totalUsers",
"type": "TYPE_INTEGER"
},
{
"name": "sessions",
"type": "TYPE_INTEGER"
},
{
"name": "bounces",
"type": "TYPE_INTEGER"
},
{
"name": "userEngagementDuration",
"type": "TYPE_SECONDS"
}
],
"rowCount": 4,
"rows": [
{
"dimensionValues": [
{
"value": "20240226"
},
{
"value": "iOS"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240224"
},
{
"value": "Windows"
}
],
"metricValues": [
{
"value": "2"
},
{
"value": "2"
},
{
"value": "2"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "Macintosh"
}
],
"metricValues": [
{
"value": "2"
},
{
"value": "3"
},
{
"value": "0"
},
{
"value": "25"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "iOS"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
},
{
"value": "4"
}
]
}
]
}
]
}

232
fixture/ga4_report_imported_pages.json Normal file
View File

@ -0,0 +1,232 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
},
{
"name": "hostName"
},
{
"name": "pagePath"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "totalUsers",
"type": "TYPE_INTEGER"
},
{
"name": "screenPageViews",
"type": "TYPE_INTEGER"
},
{
"name": "userEngagementDuration",
"type": "TYPE_SECONDS"
}
],
"rowCount": 8,
"rows": [
{
"dimensionValues": [
{
"value": "20240226"
},
{
"value": "drawer.todo.computer"
},
{
"value": "/blog/firstpost/"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240224"
},
{
"value": "drawer-4l3.pages.dev"
},
{
"value": "/"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240224"
},
{
"value": "drawer.todo.computer"
},
{
"value": "/"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "drawer.todo.computer"
},
{
"value": "/"
}
],
"metricValues": [
{
"value": "2"
},
{
"value": "3"
},
{
"value": "7"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "drawer.todo.computer"
},
{
"value": "/blog/unicode-in-elixir/"
}
],
"metricValues": [
{
"value": "2"
},
{
"value": "7"
},
{
"value": "21"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "drawer.todo.computer"
},
{
"value": "/about/"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "drawer.todo.computer"
},
{
"value": "/blog/"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "1"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "drawer.todo.computer"
},
{
"value": "/blog/firstpost/"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
}
]
}
]
}
]
}

161
fixture/ga4_report_imported_sources.json Normal file
View File

@ -0,0 +1,161 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
},
{
"name": "sessionSource"
},
{
"name": "sessionMedium"
},
{
"name": "sessionCampaignName"
},
{
"name": "sessionManualAdContent"
},
{
"name": "sessionGoogleAdsKeyword"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "totalUsers",
"type": "TYPE_INTEGER"
},
{
"name": "sessions",
"type": "TYPE_INTEGER"
},
{
"name": "bounces",
"type": "TYPE_INTEGER"
},
{
"name": "userEngagementDuration",
"type": "TYPE_SECONDS"
}
],
"rowCount": 3,
"rows": [
{
"dimensionValues": [
{
"value": "20240226"
},
{
"value": "(direct)"
},
{
"value": "(none)"
},
{
"value": "(direct)"
},
{
"value": "(not set)"
},
{
"value": "(not set)"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240224"
},
{
"value": "(direct)"
},
{
"value": "(none)"
},
{
"value": "(direct)"
},
{
"value": "(not set)"
},
{
"value": "(not set)"
}
],
"metricValues": [
{
"value": "2"
},
{
"value": "2"
},
{
"value": "2"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
},
{
"value": "(direct)"
},
{
"value": "(none)"
},
{
"value": "(direct)"
},
{
"value": "(not set)"
},
{
"value": "(not set)"
}
],
"metricValues": [
{
"value": "3"
},
{
"value": "4"
},
{
"value": "0"
},
{
"value": "29"
}
]
}
]
}
]
}

114
fixture/ga4_report_imported_visitors.json Normal file
View File

@ -0,0 +1,114 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "totalUsers",
"type": "TYPE_INTEGER"
},
{
"name": "screenPageViews",
"type": "TYPE_INTEGER"
},
{
"name": "bounces",
"type": "TYPE_INTEGER"
},
{
"name": "sessions",
"type": "TYPE_INTEGER"
},
{
"name": "userEngagementDuration",
"type": "TYPE_SECONDS"
}
],
"rowCount": 3,
"rows": [
{
"dimensionValues": [
{
"value": "20240226"
}
],
"metricValues": [
{
"value": "1"
},
{
"value": "1"
},
{
"value": "1"
},
{
"value": "1"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240224"
}
],
"metricValues": [
{
"value": "2"
},
{
"value": "2"
},
{
"value": "2"
},
{
"value": "2"
},
{
"value": "0"
}
]
},
{
"dimensionValues": [
{
"value": "20240222"
}
],
"metricValues": [
{
"value": "3"
},
{
"value": "13"
},
{
"value": "0"
},
{
"value": "4"
},
{
"value": "29"
}
]
}
]
}
]
}

38
fixture/ga4_start_date.json Normal file
View File

@ -0,0 +1,38 @@
{
"kind": "analyticsData#batchRunReports",
"reports": [
{
"dimensionHeaders": [
{
"name": "date"
}
],
"kind": "analyticsData#runReport",
"metadata": {
"currencyCode": "USD",
"timeZone": "Europe/Warsaw"
},
"metricHeaders": [
{
"name": "screenPageViews",
"type": "TYPE_INTEGER"
}
],
"rowCount": 3,
"rows": [
{
"dimensionValues": [
{
"value": "20240222"
}
],
"metricValues": [
{
"value": "13"
}
]
}
]
}
]
}

160
lib/plausible/google/api.ex
View File

@ -1,9 +1,13 @@
defmodule Plausible.Google.Api do defmodule Plausible.Google.API do
alias Plausible.Google.{ReportRequest, HTTP} @moduledoc """
use Timex API to Google services.
require Logger """
@type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()} use Timex
alias Plausible.Google.HTTP
require Logger
@search_console_scope URI.encode_www_form( @search_console_scope URI.encode_www_form(
"email https://www.googleapis.com/auth/webmasters.readonly" "email https://www.googleapis.com/auth/webmasters.readonly"
@ -17,9 +21,16 @@ defmodule Plausible.Google.Api do
Jason.encode!([site_id, redirect_to]) Jason.encode!([site_id, redirect_to])
end end
def import_authorize_url(site_id, redirect_to, legacy \\ true) do def import_authorize_url(site_id, redirect_to, opts \\ []) do
legacy = Keyword.get(opts, :legacy, true)
ga4 = Keyword.get(opts, :ga4, false)
"https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <> "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <>
Jason.encode!([site_id, redirect_to, legacy]) Jason.encode!([site_id, redirect_to, legacy, ga4])
end
def fetch_access_token!(code) do
HTTP.fetch_access_token!(code)
end end
def fetch_verified_properties(auth) do def fetch_verified_properties(auth) do
@ -53,138 +64,7 @@ defmodule Plausible.Google.Api do
end end
end end
@spec list_views(access_token :: String.t()) :: def maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do
{:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()}
@doc """
Lists Google Analytics views grouped by hostname.
"""
def list_views(access_token) do
case HTTP.list_views_for_user(access_token) do
{:ok, %{"items" => views}} ->
views = Enum.group_by(views, &view_hostname/1, &view_names/1)
{:ok, views}
error ->
error
end
end
defp view_hostname(view) do
case view do
%{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host)
_any -> "Others"
end
end
defp view_names(%{"name" => name, "id" => id}) do
{"#{id} - #{name}", id}
end
@spec get_view(access_token :: String.t(), lookup_id :: String.t()) ::
{:ok, google_analytics_view()} | {:ok, nil} | {:error, term()}
@doc """
Returns a single Google Analytics view if the user has access to it.
"""
def get_view(access_token, lookup_id) do
case list_views(access_token) do
{:ok, views} ->
view =
views
|> Map.values()
|> List.flatten()
|> Enum.find(fn {_name, id} -> id == lookup_id end)
{:ok, view}
{:error, cause} ->
{:error, cause}
end
end
@type import_auth :: {
access_token :: String.t(),
refresh_token :: String.t(),
expires_at :: String.t()
}
@per_page 7_500
@backoff_factor :timer.seconds(10)
@max_attempts 5
@spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) ::
:ok | {:error, term()}
@doc """
Imports stats from a Google Analytics UA view to a Plausible site.
This function fetches Google Analytics reports in batches of #{@per_page} per
request. The batches are then passed to persist callback.
Requests to Google Analytics can fail, and are retried at most
#{@max_attempts} times with an exponential backoff. Returns `:ok` when
importing has finished or `{:error, term()}` when a request to GA failed too
many times.
Useful links:
- [Feature documentation](https://plausible.io/docs/google-analytics-import)
- [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest)
- [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)
"""
def import_analytics(date_range, view_id, auth, persist_fn) do
with {:ok, access_token} <- maybe_refresh_token(auth) do
do_import_analytics(date_range, view_id, access_token, persist_fn)
end
end
defp do_import_analytics(date_range, view_id, access_token, persist_fn) do
Enum.reduce_while(ReportRequest.full_report(), :ok, fn report_request, :ok ->
report_request = %ReportRequest{
report_request
| date_range: date_range,
view_id: view_id,
access_token: access_token,
page_token: nil,
page_size: @per_page
}
case fetch_and_persist(report_request, persist_fn: persist_fn) do
:ok -> {:cont, :ok}
{:error, _} = error -> {:halt, error}
end
end)
end
@spec fetch_and_persist(ReportRequest.t(), Keyword.t()) ::
:ok | {:error, term()}
def fetch_and_persist(%ReportRequest{} = report_request, opts \\ []) do
persist_fn = Keyword.fetch!(opts, :persist_fn)
attempt = Keyword.get(opts, :attempt, 1)
sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
case HTTP.get_report(report_request) do
{:ok, {rows, next_page_token}} ->
:ok = persist_fn.(report_request.dataset, rows)
if next_page_token do
fetch_and_persist(
%ReportRequest{report_request | page_token: next_page_token},
opts
)
else
:ok
end
{:error, cause} ->
if attempt >= @max_attempts do
{:error, cause}
else
Process.sleep(attempt * sleep_time)
fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
end
end
end
defp maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do
with true <- needs_to_refresh_token?(auth.expires), with true <- needs_to_refresh_token?(auth.expires),
{:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token), {:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token),
changeset <- changeset <-
@ -200,7 +80,7 @@ defmodule Plausible.Google.Api do
end end
end end
defp maybe_refresh_token({access_token, refresh_token, expires_at}) do def maybe_refresh_token({access_token, refresh_token, expires_at}) do
with true <- needs_to_refresh_token?(expires_at), with true <- needs_to_refresh_token?(expires_at),
{:ok, {new_access_token, _expires_at}} <- do_refresh_token(refresh_token) do {:ok, {new_access_token, _expires_at}} <- do_refresh_token(refresh_token) do
{:ok, new_access_token} {:ok, new_access_token}

142
lib/plausible/google/ga4/api.ex Normal file
View File

@ -0,0 +1,142 @@
defmodule Plausible.Google.GA4.API do
@moduledoc """
API for Google Analytics 4.
"""
alias Plausible.Google
alias Plausible.Google.GA4
require Logger
@type import_auth :: {
access_token :: String.t(),
refresh_token :: String.t(),
expires_at :: String.t()
}
@per_page 50_000
@backoff_factor :timer.seconds(10)
@max_attempts 5
def list_properties(access_token) do
case GA4.HTTP.list_accounts_for_user(access_token) do
{:ok, %{"accountSummaries" => accounts}} ->
accounts =
accounts
|> Enum.filter(& &1["propertySummaries"])
|> Enum.map(fn account ->
{"#{account["displayName"]} (#{account["account"]})",
Enum.map(account["propertySummaries"], fn property ->
{"#{property["displayName"]} (#{property["property"]})", property["property"]}
end)}
end)
{:ok, accounts}
error ->
error
end
end
def get_property(access_token, lookup_property) do
case list_properties(access_token) do
{:ok, properties} ->
property =
properties
|> Enum.map(&elem(&1, 1))
|> List.flatten()
|> Enum.find(fn {_name, property} -> property == lookup_property end)
{:ok, property}
{:error, cause} ->
{:error, cause}
end
end
def get_analytics_start_date(access_token, property) do
GA4.HTTP.get_analytics_start_date(access_token, property)
end
def import_analytics(date_range, property, auth, persist_fn) do
Logger.debug(
"[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}"
)
with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
do_import_analytics(date_range, property, access_token, persist_fn)
end
end
defp do_import_analytics(date_range, property, access_token, persist_fn) do
Enum.reduce_while(GA4.ReportRequest.full_report(), :ok, fn report_request, :ok ->
Logger.debug(
"[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}"
)
report_request = prepare_request(report_request, date_range, property, access_token)
case fetch_and_persist(report_request, persist_fn: persist_fn) do
:ok -> {:cont, :ok}
{:error, _} = error -> {:halt, error}
end
end)
end
@spec fetch_and_persist(GA4.ReportRequest.t(), Keyword.t()) ::
:ok | {:error, term()}
def fetch_and_persist(%GA4.ReportRequest{} = report_request, opts \\ []) do
persist_fn = Keyword.fetch!(opts, :persist_fn)
attempt = Keyword.get(opts, :attempt, 1)
sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
case GA4.HTTP.get_report(report_request) do
{:ok, {rows, row_count}} ->
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Fetched #{length(rows)} rows of total #{row_count} with offset #{report_request.offset} for #{report_request.dataset}"
)
:ok = persist_fn.(report_request.dataset, rows)
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Persisted #{length(rows)} for #{report_request.dataset}"
)
if report_request.offset + @per_page < row_count do
fetch_and_persist(
%GA4.ReportRequest{report_request | offset: report_request.offset + @per_page},
opts
)
else
:ok
end
{:error, cause} ->
if attempt >= @max_attempts do
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Terminating."
)
{:error, cause}
else
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Will retry."
)
Process.sleep(attempt * sleep_time)
fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
end
end
end
defp prepare_request(report_request, date_range, property, access_token) do
%GA4.ReportRequest{
report_request
| date_range: date_range,
property: property,
access_token: access_token,
offset: 0,
limit: @per_page
}
end
end

199
lib/plausible/google/ga4/http.ex Normal file
View File

@ -0,0 +1,199 @@
defmodule Plausible.Google.GA4.HTTP do
@moduledoc """
HTTP client implementation for Google Analytics 4 API.
"""
alias Plausible.HTTPClient
require Logger
@spec get_report(Plausible.Google.GA4.ReportRequest.t()) ::
{:ok, {[map()], non_neg_integer()}} | {:error, any()}
def get_report(%Plausible.Google.GA4.ReportRequest{} = report_request) do
params = %{
requests: [
%{
property: report_request.property,
dateRanges: [
%{
startDate: report_request.date_range.first,
endDate: report_request.date_range.last
}
],
dimensions: Enum.map(report_request.dimensions, &%{name: &1}),
metrics: Enum.map(report_request.metrics, &build_metric/1),
orderBys: [
%{
dimension: %{
dimensionName: "date",
orderType: "ALPHANUMERIC"
},
desc: true
}
],
limit: report_request.limit,
offset: report_request.offset
}
]
}
url =
"#{reporting_api_url()}/v1beta/#{report_request.property}:batchRunReports"
response =
HTTPClient.impl().post(
url,
[{"Authorization", "Bearer #{report_request.access_token}"}],
params,
receive_timeout: 60_000
)
with {:ok, %{body: body}} <- response,
# File.write!("fixture/ga4_report_#{report_request.dataset}.json", Jason.encode!(body)),
{:ok, report} <- parse_report_from_response(body),
row_count <- Map.get(report, "rowCount"),
{:ok, report} <- convert_to_maps(report) do
{:ok, {report, row_count}}
else
{:error, %{reason: %{status: status, body: body}}} ->
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}"
)
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
{:error, :request_failed}
{:error, reason} ->
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}: #{inspect(reason)}"
)
{:error, :request_failed}
end
end
defp build_metric(expression) do
case String.split(expression, " = ") do
[name, expression] ->
%{
name: name,
expression: expression
}
[name] ->
%{name: name}
end
end
defp parse_report_from_response(%{"reports" => [report | _]}) do
{:ok, report}
end
defp parse_report_from_response(body) do
Sentry.Context.set_extra_context(%{google_analytics4_response: body})
Logger.error(
"Google Analytics 4: Failed to find report in response. Reason: #{inspect(body)}"
)
{:error, {:invalid_response, body}}
end
defp convert_to_maps(%{
"rows" => rows,
"dimensionHeaders" => dimension_headers,
"metricHeaders" => metric_headers
})
when is_list(rows) do
dimension_headers = Enum.map(dimension_headers, & &1["name"])
metric_headers = Enum.map(metric_headers, & &1["name"])
report =
Enum.map(rows, fn %{"dimensionValues" => dimensions, "metricValues" => metrics} ->
dimension_values = Enum.map(dimensions, & &1["value"])
metric_values = Enum.map(metrics, & &1["value"])
metrics = Enum.zip(metric_headers, metric_values)
dimensions = Enum.zip(dimension_headers, dimension_values)
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
end)
{:ok, report}
end
defp convert_to_maps(response) do
Logger.error(
"Google Analytics 4: Failed to read report in response. Reason: #{inspect(response)}"
)
Sentry.Context.set_extra_context(%{google_analytics4_response: response})
{:error, {:invalid_response, response}}
end
def list_accounts_for_user(access_token) do
url = "#{admin_api_url()}/v1beta/accountSummaries"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.impl().get(url, headers) do
{:ok, %Finch.Response{body: body, status: 200}} ->
{:ok, body}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %HTTPClient.Non200Error{} = error} ->
Sentry.capture_message("Error listing Google accounts for user", extra: %{error: error})
{:error, :unknown}
end
end
@earliest_valid_date "2015-08-14"
def get_analytics_start_date(access_token, property) do
params = %{
requests: [
%{
property: "#{property}",
dateRanges: [
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
],
dimensions: [%{name: "date"}],
metrics: [%{name: "screenPageViews"}],
orderBys: [
%{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: false}
],
limit: 1
}
]
}
url = "#{reporting_api_url()}/v1beta/#{property}:batchRunReports"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.impl().post(url, headers, params) do
{:ok, %Finch.Response{body: body, status: 200}} ->
report = List.first(body["reports"])
date =
case report["rows"] do
[%{"dimensionValues" => [%{"value" => date_str}]}] ->
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
_ ->
nil
end
{:ok, date}
{:error, %{reason: %Finch.Response{body: body}}} ->
Sentry.capture_message("Error fetching GA4 start date", extra: %{body: inspect(body)})
{:error, body}
{:error, %{reason: reason} = e} ->
Sentry.capture_message("Error fetching GA4 start date", extra: %{error: inspect(e)})
{:error, reason}
end
end
defp reporting_api_url, do: "https://analyticsdata.googleapis.com"
defp admin_api_url, do: "https://analyticsadmin.googleapis.com"
end

122
lib/plausible/google/ga4/report_request.ex Normal file
View File

@ -0,0 +1,122 @@
defmodule Plausible.Google.GA4.ReportRequest do
@moduledoc """
Report request struct for Google Analytics 4 API
"""
defstruct [
:dataset,
:dimensions,
:metrics,
:date_range,
:property,
:access_token,
:offset,
:limit
]
@type t() :: %__MODULE__{
dataset: String.t(),
dimensions: [String.t()],
metrics: [String.t()],
date_range: Date.Range.t(),
property: term(),
access_token: String.t(),
offset: non_neg_integer(),
limit: non_neg_integer()
}
def full_report do
[
%__MODULE__{
dataset: "imported_visitors",
dimensions: ["date"],
metrics: [
"totalUsers",
"screenPageViews",
"bounces = sessions - engagedSessions",
"sessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_sources",
dimensions: [
"date",
"sessionSource",
"sessionMedium",
"sessionCampaignName",
"sessionManualAdContent",
"sessionGoogleAdsKeyword"
],
metrics: [
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_pages",
dimensions: ["date", "hostName", "pagePath"],
# NOTE: no exits as GA4 DATA API does not provide that metric
metrics: ["totalUsers", "screenPageViews", "userEngagementDuration"]
},
%__MODULE__{
dataset: "imported_entry_pages",
dimensions: ["date", "landingPage"],
metrics: [
"totalUsers",
"sessions",
"userEngagementDuration",
"bounces = sessions - engagedSessions"
]
},
# NOTE: Skipping for now as there's no dimension directly mapping to exit page path
# %__MODULE__{
# dataset: "imported_exit_pages",
# dimensions: ["date", "ga:exitPagePath"],
# metrics: ["totalUsers", "sessions"]
# },
%__MODULE__{
dataset: "imported_locations",
dimensions: ["date", "countryId", "region", "city"],
metrics: [
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_devices",
dimensions: ["date", "deviceCategory"],
metrics: [
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_browsers",
dimensions: ["date", "browser"],
metrics: [
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
},
%__MODULE__{
dataset: "imported_operating_systems",
dimensions: ["date", "operatingSystem"],
metrics: [
"totalUsers",
"sessions",
"bounces = sessions - engagedSessions",
"userEngagementDuration"
]
}
]
end
end

159
lib/plausible/google/http.ex
View File

@ -2,97 +2,6 @@ defmodule Plausible.Google.HTTP do
require Logger require Logger
alias Plausible.HTTPClient alias Plausible.HTTPClient
@spec get_report(Plausible.Google.ReportRequest.t()) ::
{:ok, {[map()], String.t() | nil}} | {:error, any()}
def get_report(%Plausible.Google.ReportRequest{} = report_request) do
params = %{
reportRequests: [
%{
viewId: report_request.view_id,
dateRanges: [
%{
startDate: report_request.date_range.first,
endDate: report_request.date_range.last
}
],
dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}),
metrics: Enum.map(report_request.metrics, &%{expression: &1}),
hideTotals: true,
hideValueRanges: true,
orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],
pageSize: report_request.page_size,
pageToken: report_request.page_token
}
]
}
response =
HTTPClient.impl().post(
"#{reporting_api_url()}/v4/reports:batchGet",
[{"Authorization", "Bearer #{report_request.access_token}"}],
params,
receive_timeout: 60_000
)
with {:ok, %{body: body}} <- response,
{:ok, report} <- parse_report_from_response(body),
token <- Map.get(report, "nextPageToken"),
{:ok, report} <- convert_to_maps(report) do
{:ok, {report, token}}
else
{:error, %{reason: %{status: status, body: body}}} ->
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
{:error, :request_failed}
{:error, _reason} ->
{:error, :request_failed}
end
end
defp parse_report_from_response(body) do
with %{"reports" => [report | _]} <- body do
{:ok, report}
else
_ ->
Sentry.Context.set_extra_context(%{google_analytics_response: body})
Logger.error(
"Google Analytics: Failed to find report in response. Reason: #{inspect(body)}"
)
{:error, {:invalid_response, body}}
end
end
defp convert_to_maps(%{
"data" => %{} = data,
"columnHeader" => %{
"dimensions" => dimension_headers,
"metricHeader" => %{"metricHeaderEntries" => metric_headers}
}
}) do
metric_headers = Enum.map(metric_headers, & &1["name"])
rows = Map.get(data, "rows", [])
report =
Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} ->
metrics = Enum.zip(metric_headers, metrics)
dimensions = Enum.zip(dimension_headers, dimensions)
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
end)
{:ok, report}
end
defp convert_to_maps(response) do
Logger.error(
"Google Analytics: Failed to read report in response. Reason: #{inspect(response)}"
)
Sentry.Context.set_extra_context(%{google_analytics_response: response})
{:error, {:invalid_response, response}}
end
def list_sites(access_token) do def list_sites(access_token) do
url = "#{api_url()}/webmasters/v3/sites" url = "#{api_url()}/webmasters/v3/sites"
headers = [{"Content-Type", "application/json"}, {"Authorization", "Bearer #{access_token}"}] headers = [{"Content-Type", "application/json"}, {"Authorization", "Bearer #{access_token}"}]
@ -113,7 +22,7 @@ defmodule Plausible.Google.HTTP do
end end
end end
def fetch_access_token(code) do def fetch_access_token!(code) do
url = "#{api_url()}/oauth2/v4/token" url = "#{api_url()}/oauth2/v4/token"
headers = [{"Content-Type", "application/x-www-form-urlencoded"}] headers = [{"Content-Type", "application/x-www-form-urlencoded"}]
@ -130,24 +39,6 @@ defmodule Plausible.Google.HTTP do
response.body response.body
end end
def list_views_for_user(access_token) do
url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.impl().get(url, headers) do
{:ok, %Finch.Response{body: body, status: 200}} ->
{:ok, body}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %HTTPClient.Non200Error{} = error} ->
Sentry.capture_message("Error listing GA views for user", extra: %{error: error})
{:error, :unknown}
end
end
def list_stats(access_token, property, date_range, limit, page \\ nil) do def list_stats(access_token, property, date_range, limit, page \\ nil) do
property = URI.encode_www_form(property) property = URI.encode_www_form(property)
@ -215,57 +106,9 @@ defmodule Plausible.Google.HTTP do
end end
end end
@earliest_valid_date "2005-01-01"
def get_analytics_start_date(view_id, access_token) do
params = %{
reportRequests: [
%{
viewId: view_id,
dateRanges: [
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
],
dimensions: [%{name: "ga:date", histogramBuckets: []}],
metrics: [%{expression: "ga:pageviews"}],
hideTotals: true,
hideValueRanges: true,
orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}],
pageSize: 1
}
]
}
url = "#{reporting_api_url()}/v4/reports:batchGet"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.post(url, headers, params) do
{:ok, %Finch.Response{body: body, status: 200}} ->
report = List.first(body["reports"])
date =
case report["data"]["rows"] do
[%{"dimensions" => [date_str]}] ->
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
_ ->
nil
end
{:ok, date}
{:error, %{reason: %Finch.Response{body: body}}} ->
Sentry.capture_message("Error fetching Google view ID", extra: %{body: inspect(body)})
{:error, body}
{:error, %{reason: reason} = e} ->
Sentry.capture_message("Error fetching Google view ID", extra: %{error: inspect(e)})
{:error, reason}
end
end
defp config, do: Application.get_env(:plausible, :google) defp config, do: Application.get_env(:plausible, :google)
defp client_id, do: Keyword.fetch!(config(), :client_id) defp client_id, do: Keyword.fetch!(config(), :client_id)
defp client_secret, do: Keyword.fetch!(config(), :client_secret) defp client_secret, do: Keyword.fetch!(config(), :client_secret)
defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url)
defp api_url, do: Keyword.fetch!(config(), :api_url) defp api_url, do: Keyword.fetch!(config(), :api_url)
defp redirect_uri, do: PlausibleWeb.Endpoint.url() <> "/auth/google/callback" defp redirect_uri, do: PlausibleWeb.Endpoint.url() <> "/auth/google/callback"
end end

146
lib/plausible/google/ua/api.ex Normal file
View File

@ -0,0 +1,146 @@
defmodule Plausible.Google.UA.API do
@moduledoc """
API for Universal Analytics
"""
alias Plausible.Google
alias Plausible.Google.UA
@type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()}
@type import_auth :: {
access_token :: String.t(),
refresh_token :: String.t(),
expires_at :: String.t()
}
@per_page 7_500
@backoff_factor :timer.seconds(10)
@max_attempts 5
@spec list_views(access_token :: String.t()) ::
{:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()}
@doc """
Lists Google Analytics views grouped by hostname.
"""
def list_views(access_token) do
case UA.HTTP.list_views_for_user(access_token) do
{:ok, %{"items" => views}} ->
views = Enum.group_by(views, &view_hostname/1, &view_names/1)
{:ok, views}
error ->
error
end
end
@spec get_view(access_token :: String.t(), lookup_id :: String.t()) ::
{:ok, google_analytics_view()} | {:ok, nil} | {:error, term()}
@doc """
Returns a single Google Analytics view if the user has access to it.
"""
def get_view(access_token, lookup_id) do
case list_views(access_token) do
{:ok, views} ->
view =
views
|> Map.values()
|> List.flatten()
|> Enum.find(fn {_name, id} -> id == lookup_id end)
{:ok, view}
{:error, cause} ->
{:error, cause}
end
end
def get_analytics_start_date(access_token, view_id) do
UA.HTTP.get_analytics_start_date(access_token, view_id)
end
@spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) ::
:ok | {:error, term()}
@doc """
Imports stats from a Google Analytics UA view to a Plausible site.
This function fetches Google Analytics reports in batches of #{@per_page} per
request. The batches are then passed to persist callback.
Requests to Google Analytics can fail, and are retried at most
#{@max_attempts} times with an exponential backoff. Returns `:ok` when
importing has finished or `{:error, term()}` when a request to GA failed too
many times.
Useful links:
- [Feature documentation](https://plausible.io/docs/google-analytics-import)
- [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest)
- [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)
"""
def import_analytics(date_range, view_id, auth, persist_fn) do
with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
do_import_analytics(date_range, view_id, access_token, persist_fn)
end
end
defp do_import_analytics(date_range, view_id, access_token, persist_fn) do
Enum.reduce_while(UA.ReportRequest.full_report(), :ok, fn report_request, :ok ->
report_request = %UA.ReportRequest{
report_request
| date_range: date_range,
view_id: view_id,
access_token: access_token,
page_token: nil,
page_size: @per_page
}
case fetch_and_persist(report_request, persist_fn: persist_fn) do
:ok -> {:cont, :ok}
{:error, _} = error -> {:halt, error}
end
end)
end
@spec fetch_and_persist(UA.ReportRequest.t(), Keyword.t()) ::
:ok | {:error, term()}
def fetch_and_persist(%UA.ReportRequest{} = report_request, opts \\ []) do
persist_fn = Keyword.fetch!(opts, :persist_fn)
attempt = Keyword.get(opts, :attempt, 1)
sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
case UA.HTTP.get_report(report_request) do
{:ok, {rows, next_page_token}} ->
:ok = persist_fn.(report_request.dataset, rows)
if next_page_token do
fetch_and_persist(
%UA.ReportRequest{report_request | page_token: next_page_token},
opts
)
else
:ok
end
{:error, cause} ->
if attempt >= @max_attempts do
{:error, cause}
else
Process.sleep(attempt * sleep_time)
fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
end
end
end
defp view_hostname(view) do
case view do
%{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host)
_any -> "Others"
end
end
defp view_names(%{"name" => name, "id" => id}) do
{"#{id} - #{name}", id}
end
end

167
lib/plausible/google/ua/http.ex Normal file
View File

@ -0,0 +1,167 @@
defmodule Plausible.Google.UA.HTTP do
@moduledoc """
HTTP client implementation for Universal Analytics API.
"""
require Logger
alias Plausible.HTTPClient
@spec get_report(Plausible.Google.UA.ReportRequest.t()) ::
{:ok, {[map()], String.t() | nil}} | {:error, any()}
def get_report(%Plausible.Google.UA.ReportRequest{} = report_request) do
params = %{
reportRequests: [
%{
viewId: report_request.view_id,
dateRanges: [
%{
startDate: report_request.date_range.first,
endDate: report_request.date_range.last
}
],
dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}),
metrics: Enum.map(report_request.metrics, &%{expression: &1}),
hideTotals: true,
hideValueRanges: true,
orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],
pageSize: report_request.page_size,
pageToken: report_request.page_token
}
]
}
response =
HTTPClient.impl().post(
"#{reporting_api_url()}/v4/reports:batchGet",
[{"Authorization", "Bearer #{report_request.access_token}"}],
params,
receive_timeout: 60_000
)
with {:ok, %{body: body}} <- response,
{:ok, report} <- parse_report_from_response(body),
token <- Map.get(report, "nextPageToken"),
{:ok, report} <- convert_to_maps(report) do
{:ok, {report, token}}
else
{:error, %{reason: %{status: status, body: body}}} ->
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
{:error, :request_failed}
{:error, _reason} ->
{:error, :request_failed}
end
end
defp parse_report_from_response(%{"reports" => [report | _]}) do
{:ok, report}
end
defp parse_report_from_response(body) do
Sentry.Context.set_extra_context(%{universal_analytics_response: body})
Logger.error(
"Universal Analytics: Failed to find report in response. Reason: #{inspect(body)}"
)
{:error, {:invalid_response, body}}
end
defp convert_to_maps(%{
"data" => %{} = data,
"columnHeader" => %{
"dimensions" => dimension_headers,
"metricHeader" => %{"metricHeaderEntries" => metric_headers}
}
}) do
metric_headers = Enum.map(metric_headers, & &1["name"])
rows = Map.get(data, "rows", [])
report =
Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} ->
metrics = Enum.zip(metric_headers, metrics)
dimensions = Enum.zip(dimension_headers, dimensions)
%{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
end)
{:ok, report}
end
defp convert_to_maps(response) do
Logger.error(
"Universal Analytics: Failed to read report in response. Reason: #{inspect(response)}"
)
Sentry.Context.set_extra_context(%{universal_analytics_response: response})
{:error, {:invalid_response, response}}
end
def list_views_for_user(access_token) do
url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.impl().get(url, headers) do
{:ok, %Finch.Response{body: body, status: 200}} ->
{:ok, body}
{:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
{:error, :authentication_failed}
{:error, %HTTPClient.Non200Error{} = error} ->
Sentry.capture_message("Error listing GA views for user", extra: %{error: error})
{:error, :unknown}
end
end
@earliest_valid_date "2005-01-01"
def get_analytics_start_date(access_token, view_id) do
params = %{
reportRequests: [
%{
viewId: view_id,
dateRanges: [
%{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
],
dimensions: [%{name: "ga:date", histogramBuckets: []}],
metrics: [%{expression: "ga:pageviews"}],
hideTotals: true,
hideValueRanges: true,
orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}],
pageSize: 1
}
]
}
url = "#{reporting_api_url()}/v4/reports:batchGet"
headers = [{"Authorization", "Bearer #{access_token}"}]
case HTTPClient.post(url, headers, params) do
{:ok, %Finch.Response{body: body, status: 200}} ->
report = List.first(body["reports"])
date =
case report["data"]["rows"] do
[%{"dimensions" => [date_str]}] ->
Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
_ ->
nil
end
{:ok, date}
{:error, %{reason: %Finch.Response{body: body}}} ->
Sentry.capture_message("Error fetching UA start date", extra: %{body: inspect(body)})
{:error, body}
{:error, %{reason: reason} = e} ->
Sentry.capture_message("Error fetching UA start date", extra: %{error: inspect(e)})
{:error, reason}
end
end
defp config, do: Application.get_env(:plausible, :google)
defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url)
defp api_url, do: Keyword.fetch!(config(), :api_url)
end

6
lib/plausible/google/report_request.ex → lib/plausible/google/ua/report_request.ex
View File

@ -1,4 +1,8 @@
defmodule Plausible.Google.ReportRequest do defmodule Plausible.Google.UA.ReportRequest do
@moduledoc """
Report request struct for Universal Analytics API
"""
defstruct [ defstruct [
:dataset, :dataset,
:dimensions, :dimensions,

17
lib/plausible/imported/buffer.ex
View File

@ -8,12 +8,13 @@ defmodule Plausible.Imported.Buffer do
use GenServer use GenServer
require Logger require Logger
def start_link do def start_link(opts \\ []) do
GenServer.start_link(__MODULE__, nil) GenServer.start_link(__MODULE__, opts)
end end
def init(_opts) do def init(opts) do
{:ok, %{buffers: %{}}} flush_interval = Keyword.get(opts, :flush_interval_ms, 1000)
{:ok, %{flush_interval: flush_interval, buffers: %{}}}
end end
@spec insert_many(pid(), term(), [map()]) :: :ok @spec insert_many(pid(), term(), [map()]) :: :ok
@ -68,14 +69,14 @@ defmodule Plausible.Imported.Buffer do
def handle_call(:flush_all_buffers, _from, state) do def handle_call(:flush_all_buffers, _from, state) do
Enum.each(state.buffers, fn {table_name, records} -> Enum.each(state.buffers, fn {table_name, records} ->
flush_buffer(records, table_name) flush_buffer(records, table_name, state.flush_interval)
end) end)
{:reply, :ok, put_in(state.buffers, %{})} {:reply, :ok, put_in(state.buffers, %{})}
end end
def handle_continue({:flush, table_name}, state) do def handle_continue({:flush, table_name}, state) do
flush_buffer(state.buffers[table_name], table_name) flush_buffer(state.buffers[table_name], table_name, state.flush_interval)
{:noreply, put_in(state.buffers[table_name], [])} {:noreply, put_in(state.buffers[table_name], [])}
end end
@ -85,10 +86,10 @@ defmodule Plausible.Imported.Buffer do
|> Keyword.fetch!(:max_buffer_size) |> Keyword.fetch!(:max_buffer_size)
end end
defp flush_buffer(records, table_name) do defp flush_buffer(records, table_name, flush_interval) do
# Clickhouse does not recommend sending more than 1 INSERT operation per second, and this # Clickhouse does not recommend sending more than 1 INSERT operation per second, and this
# sleep call slows down the flushing # sleep call slows down the flushing
Process.sleep(1000) Process.sleep(flush_interval)
Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer") Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer")
insert_all(table_name, records) insert_all(table_name, records)

253
lib/plausible/imported/google_analytics4.ex Normal file
View File

@ -0,0 +1,253 @@
defmodule Plausible.Imported.GoogleAnalytics4 do
@moduledoc """
Import implementation for Google Analytics 4.
"""
use Plausible.Imported.Importer
@missing_values ["(none)", "(not set)", "(not provided)", "(other)"]
@impl true
def name(), do: :google_analytics_4
@impl true
def label(), do: "Google Analytics 4"
@impl true
def email_template(), do: "google_analytics_import.html"
@impl true
def parse_args(
%{"property" => property, "start_date" => start_date, "end_date" => end_date} = args
) do
start_date = Date.from_iso8601!(start_date)
end_date = Date.from_iso8601!(end_date)
date_range = Date.range(start_date, end_date)
auth = {
Map.fetch!(args, "access_token"),
Map.fetch!(args, "refresh_token"),
Map.fetch!(args, "token_expires_at")
}
[
property: property,
date_range: date_range,
auth: auth
]
end
@doc """
Imports stats from a Google Analytics 4 property to a Plausible site.
This function fetches Google Analytics 4 reports which are then passed in batches
to Clickhouse by the `Plausible.Imported.Buffer` process.
"""
@impl true
def import_data(site_import, opts) do
date_range = Keyword.fetch!(opts, :date_range)
property = Keyword.fetch!(opts, :property)
auth = Keyword.fetch!(opts, :auth)
flush_interval_ms = Keyword.get(opts, :flush_interval_ms, 1000)
{:ok, buffer} = Plausible.Imported.Buffer.start_link(flush_interval_ms: flush_interval_ms)
persist_fn = fn table, rows ->
records = from_report(rows, site_import.site_id, site_import.id, table)
Plausible.Imported.Buffer.insert_many(buffer, table, records)
end
try do
Plausible.Google.GA4.API.import_analytics(date_range, property, auth, persist_fn)
after
Plausible.Imported.Buffer.flush(buffer)
Plausible.Imported.Buffer.stop(buffer)
end
end
def from_report(nil, _site_id, _import_id, _metric), do: nil
def from_report(data, site_id, import_id, table) do
Enum.reduce(data, [], fn row, acc ->
if Map.get(row.dimensions, "date") in @missing_values do
acc
else
[new_from_report(site_id, import_id, table, row) | acc]
end
end)
end
defp parse_number(nr) do
{float, ""} = Float.parse(nr)
round(float)
end
defp new_from_report(site_id, import_id, "imported_visitors", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
end
defp new_from_report(site_id, import_id, "imported_sources", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(),
utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(),
utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(),
utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(),
utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
end
defp new_from_report(site_id, import_id, "imported_pages", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""),
page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
# NOTE: no exits metric in GA4 API currently
exits: 0,
time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
end
defp new_from_report(site_id, import_id, "imported_entry_pages", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
entry_page: row.dimensions |> Map.fetch!("landingPage"),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
}
end
# NOTE: no exit pages metrics in GA4 API available for now
# defp new_from_report(site_id, import_id, "imported_exit_pages", row) do
# %{
# site_id: site_id,
# import_id: import_id,
# date: get_date(row),
# exit_page: Map.fetch!(row.dimensions, "exitPage"),
# visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
# exits: row.metrics |> Map.fetch!("sessions") |> parse_number()
# }
# end
defp new_from_report(site_id, import_id, "imported_locations", row) do
country_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("")
city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("")
city_data = Location.get_city(city_name, country_code)
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
country: country_code,
region: row.dimensions |> Map.fetch!("region") |> default_if_missing(""),
city: city_data && city_data.id,
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
end
defp new_from_report(site_id, import_id, "imported_devices", row) do
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
end
@browser_google_to_plausible %{
"User-Agent:Opera" => "Opera",
"Mozilla Compatible Agent" => "Mobile App",
"Android Webview" => "Mobile App",
"Android Browser" => "Mobile App",
"Safari (in-app)" => "Mobile App",
"User-Agent: Mozilla" => "Firefox",
"(not set)" => ""
}
defp new_from_report(site_id, import_id, "imported_browsers", row) do
browser = Map.fetch!(row.dimensions, "browser")
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
browser: Map.get(@browser_google_to_plausible, browser, browser),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
end
@os_google_to_plausible %{
"Macintosh" => "Mac",
"Linux" => "GNU/Linux",
"(not set)" => ""
}
defp new_from_report(site_id, import_id, "imported_operating_systems", row) do
os = Map.fetch!(row.dimensions, "operatingSystem")
%{
site_id: site_id,
import_id: import_id,
date: get_date(row),
operating_system: Map.get(@os_google_to_plausible, os, os),
visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
}
end
defp get_date(%{dimensions: %{"date" => date}}) do
date
|> Timex.parse!("%Y%m%d", :strftime)
|> NaiveDateTime.to_date()
end
defp default_if_missing(value, default \\ nil)
defp default_if_missing(value, default) when value in @missing_values, do: default
defp default_if_missing(value, _default), do: value
defp parse_referrer(nil), do: nil
defp parse_referrer("(direct)"), do: nil
defp parse_referrer("google"), do: "Google"
defp parse_referrer("bing"), do: "Bing"
defp parse_referrer("duckduckgo"), do: "DuckDuckGo"
defp parse_referrer(ref) do
RefInspector.parse("https://" <> ref)
|> PlausibleWeb.RefInspector.parse()
end
end

1
lib/plausible/imported/import_sources.ex
View File

@ -4,6 +4,7 @@ defmodule Plausible.Imported.ImportSources do
""" """
@sources [ @sources [
Plausible.Imported.GoogleAnalytics4,
Plausible.Imported.UniversalAnalytics, Plausible.Imported.UniversalAnalytics,
Plausible.Imported.NoopImporter, Plausible.Imported.NoopImporter,
Plausible.Imported.CSVImporter Plausible.Imported.CSVImporter

2
lib/plausible/imported/universal_analytics.ex
View File

@ -102,7 +102,7 @@ defmodule Plausible.Imported.UniversalAnalytics do
end end
try do try do
Plausible.Google.Api.import_analytics(date_range, view_id, auth, persist_fn) Plausible.Google.UA.API.import_analytics(date_range, view_id, auth, persist_fn)
after after
Plausible.Imported.Buffer.flush(buffer) Plausible.Imported.Buffer.flush(buffer)
Plausible.Imported.Buffer.stop(buffer) Plausible.Imported.Buffer.stop(buffer)

49
lib/plausible_web/controllers/auth_controller.ex
View File

@ -697,13 +697,16 @@ defmodule PlausibleWeb.AuthController do
end end
def google_auth_callback(conn, %{"error" => error, "state" => state} = params) do def google_auth_callback(conn, %{"error" => error, "state" => state} = params) do
[site_id, _redirected_to, legacy] = [site_id, _redirected_to, legacy, _ga4] =
case Jason.decode!(state) do case Jason.decode!(state) do
[site_id, redirect_to] -> [site_id, redirect_to] ->
[site_id, redirect_to, true] [site_id, redirect_to, true, false]
[site_id, redirect_to, legacy] -> [site_id, redirect_to, legacy] ->
[site_id, redirect_to, legacy] [site_id, redirect_to, legacy, false]
[site_id, redirect_to, legacy, ga4] ->
[site_id, redirect_to, legacy, ga4]
end end
site = Repo.get(Plausible.Site, site_id) site = Repo.get(Plausible.Site, site_id)
@ -745,15 +748,18 @@ defmodule PlausibleWeb.AuthController do
end end
def google_auth_callback(conn, %{"code" => code, "state" => state}) do def google_auth_callback(conn, %{"code" => code, "state" => state}) do
res = Plausible.Google.HTTP.fetch_access_token(code) res = Plausible.Google.API.fetch_access_token!(code)
[site_id, redirect_to, legacy] = [site_id, redirect_to, legacy, ga4] =
case Jason.decode!(state) do case Jason.decode!(state) do
[site_id, redirect_to] -> [site_id, redirect_to] ->
[site_id, redirect_to, true] [site_id, redirect_to, true, false]
[site_id, redirect_to, legacy] -> [site_id, redirect_to, legacy] ->
[site_id, redirect_to, legacy] [site_id, redirect_to, legacy, false]
[site_id, redirect_to, legacy, ga4] ->
[site_id, redirect_to, legacy, ga4]
end end
site = Repo.get(Plausible.Site, site_id) site = Repo.get(Plausible.Site, site_id)
@ -761,15 +767,26 @@ defmodule PlausibleWeb.AuthController do
case redirect_to do case redirect_to do
"import" -> "import" ->
redirect(conn, if ga4 do
external: redirect(conn,
Routes.site_path(conn, :import_from_google_view_id_form, site.domain, external:
access_token: res["access_token"], Routes.google_analytics4_path(conn, :property_form, site.domain,
refresh_token: res["refresh_token"], access_token: res["access_token"],
expires_at: NaiveDateTime.to_iso8601(expires_at), refresh_token: res["refresh_token"],
legacy: legacy expires_at: NaiveDateTime.to_iso8601(expires_at)
) )
) )
else
redirect(conn,
external:
Routes.universal_analytics_path(conn, :view_id_form, site.domain,
access_token: res["access_token"],
refresh_token: res["refresh_token"],
expires_at: NaiveDateTime.to_iso8601(expires_at),
legacy: legacy
)
)
end
_ -> _ ->
id_token = res["id_token"] id_token = res["id_token"]

143
lib/plausible_web/controllers/google_analytics4_controller.ex Normal file
View File

@ -0,0 +1,143 @@
defmodule PlausibleWeb.GoogleAnalytics4Controller do
use PlausibleWeb, :controller
plug(PlausibleWeb.RequireAccountPlug)
plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin])
def property_form(conn, %{
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at
}) do
redirect_route = Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain)
case Plausible.Google.GA4.API.list_properties(access_token) do
{:ok, properties} ->
conn
|> assign(:skip_plausible_tracking, true)
|> render("property_form.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: conn.assigns.site,
properties: properties,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
{:error, :authentication_failed} ->
conn
|> put_flash(
:error,
"We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again."
)
|> redirect(external: redirect_route)
{:error, _any} ->
conn
|> put_flash(
:error,
"We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance."
)
|> redirect(external: redirect_route)
end
end
def property(conn, %{
"property" => property,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at
}) do
site = conn.assigns.site
start_date = Plausible.Google.GA4.API.get_analytics_start_date(access_token, property)
case start_date do
{:ok, nil} ->
{:ok, properties} = Plausible.Google.GA4.API.list_properties(access_token)
conn
|> assign(:skip_plausible_tracking, true)
|> render("property_form.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: site,
properties: properties,
selected_property_error: "No data found. Nothing to import",
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
{:ok, _date} ->
redirect(conn,
to:
Routes.google_analytics4_path(conn, :confirm, site.domain,
property: property,
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at
)
)
end
end
def confirm(conn, %{
"property" => property,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at
}) do
site = conn.assigns.site
start_date = Plausible.Google.GA4.API.get_analytics_start_date(access_token, property)
end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)
{:ok, {property_name, property}} =
Plausible.Google.GA4.API.get_property(access_token, property)
conn
|> assign(:skip_plausible_tracking, true)
|> render("confirm.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: site,
selected_property: property,
selected_property_name: property_name,
start_date: start_date,
end_date: end_date,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
end
def import(conn, %{
"property" => property,
"start_date" => start_date,
"end_date" => end_date,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at
}) do
site = conn.assigns.site
current_user = conn.assigns.current_user
redirect_route = Routes.site_path(conn, :settings_imports_exports, site.domain)
{:ok, _} =
Plausible.Imported.GoogleAnalytics4.new_import(
site,
current_user,
property: property,
start_date: start_date,
end_date: end_date,
access_token: access_token,
refresh_token: refresh_token,
token_expires_at: expires_at
)
conn
|> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
|> redirect(external: redirect_route)
end
end

194
lib/plausible_web/controllers/site_controller.ex
View File

@ -232,7 +232,7 @@ defmodule PlausibleWeb.SiteController do
search_console_domains = search_console_domains =
if site.google_auth do if site.google_auth do
Plausible.Google.Api.fetch_verified_properties(site.google_auth) Plausible.Google.API.fetch_verified_properties(site.google_auth)
end end
imported_pageviews = imported_pageviews =
@ -641,198 +641,6 @@ defmodule PlausibleWeb.SiteController do
end end
end end
def import_from_google_user_metric_notice(conn, %{
"view_id" => view_id,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
}) do
site = conn.assigns[:site]
conn
|> assign(:skip_plausible_tracking, true)
|> render("import_from_google_user_metric_form.html",
site: site,
view_id: view_id,
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
end
def import_from_google_view_id_form(conn, %{
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
}) do
redirect_route =
if legacy == "true" do
Routes.site_path(conn, :settings_integrations, conn.assigns.site.domain)
else
Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain)
end
case Plausible.Google.Api.list_views(access_token) do
{:ok, view_ids} ->
conn
|> assign(:skip_plausible_tracking, true)
|> render("import_from_google_view_id_form.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: conn.assigns.site,
view_ids: view_ids,
legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
{:error, :authentication_failed} ->
conn
|> put_flash(
:error,
"We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again."
)
|> redirect(external: redirect_route)
{:error, _any} ->
conn
|> put_flash(
:error,
"We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance."
)
|> redirect(external: redirect_route)
end
end
# see https://stackoverflow.com/a/57416769
@google_analytics_new_user_metric_date ~D[2016-08-24]
def import_from_google_view_id(conn, %{
"view_id" => view_id,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
}) do
site = conn.assigns[:site]
start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)
case start_date do
{:ok, nil} ->
site = conn.assigns[:site]
{:ok, view_ids} = Plausible.Google.Api.list_views(access_token)
conn
|> assign(:skip_plausible_tracking, true)
|> render("import_from_google_view_id_form.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: site,
view_ids: view_ids,
selected_view_id_error: "No data found. Nothing to import",
legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
{:ok, date} ->
if Timex.before?(date, @google_analytics_new_user_metric_date) do
redirect(conn,
to:
Routes.site_path(conn, :import_from_google_user_metric_notice, site.domain,
view_id: view_id,
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
legacy: legacy
)
)
else
redirect(conn,
to:
Routes.site_path(conn, :import_from_google_confirm, site.domain,
view_id: view_id,
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
legacy: legacy
)
)
end
end
end
def import_from_google_confirm(conn, %{
"view_id" => view_id,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
}) do
site = conn.assigns[:site]
start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)
end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)
{:ok, {view_name, view_id}} = Plausible.Google.Api.get_view(access_token, view_id)
conn
|> assign(:skip_plausible_tracking, true)
|> render("import_from_google_confirm.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: site,
selected_view_id: view_id,
selected_view_id_name: view_name,
start_date: start_date,
end_date: end_date,
legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
end
def import_from_google(conn, %{
"view_id" => view_id,
"start_date" => start_date,
"end_date" => end_date,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
}) do
site = conn.assigns.site
current_user = conn.assigns.current_user
redirect_route =
if legacy == "true" do
Routes.site_path(conn, :settings_integrations, site.domain)
else
Routes.site_path(conn, :settings_imports_exports, site.domain)
end
{:ok, _} =
Plausible.Imported.UniversalAnalytics.new_import(
site,
current_user,
view_id: view_id,
start_date: start_date,
end_date: end_date,
access_token: access_token,
refresh_token: refresh_token,
token_expires_at: expires_at,
legacy: legacy == "true"
)
conn
|> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
|> redirect(external: redirect_route)
end
def forget_import(conn, %{"import_id" => import_id}) do def forget_import(conn, %{"import_id" => import_id}) do
site = conn.assigns.site site = conn.assigns.site

198
lib/plausible_web/controllers/universal_analytics_controller.ex Normal file
View File

@ -0,0 +1,198 @@
defmodule PlausibleWeb.UniversalAnalyticsController do
use PlausibleWeb, :controller
plug(PlausibleWeb.RequireAccountPlug)
plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin])
def user_metric_notice(conn, %{
"view_id" => view_id,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
}) do
site = conn.assigns.site
conn
|> assign(:skip_plausible_tracking, true)
|> render("user_metric_form.html",
site: site,
view_id: view_id,
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
end
def view_id_form(conn, %{
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
}) do
redirect_route =
if legacy == "true" do
Routes.site_path(conn, :settings_integrations, conn.assigns.site.domain)
else
Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain)
end
case Plausible.Google.UA.API.list_views(access_token) do
{:ok, view_ids} ->
conn
|> assign(:skip_plausible_tracking, true)
|> render("view_id_form.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: conn.assigns.site,
view_ids: view_ids,
legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
{:error, :authentication_failed} ->
conn
|> put_flash(
:error,
"We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again."
)
|> redirect(external: redirect_route)
{:error, _any} ->
conn
|> put_flash(
:error,
"We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance."
)
|> redirect(external: redirect_route)
end
end
# see https://stackoverflow.com/a/57416769
@google_analytics_new_user_metric_date ~D[2016-08-24]
def view_id(conn, %{
"view_id" => view_id,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
}) do
site = conn.assigns.site
start_date = Plausible.Google.UA.API.get_analytics_start_date(access_token, view_id)
case start_date do
{:ok, nil} ->
{:ok, view_ids} = Plausible.Google.UA.API.list_views(access_token)
conn
|> assign(:skip_plausible_tracking, true)
|> render("view_id_form.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: site,
view_ids: view_ids,
selected_view_id_error: "No data found. Nothing to import",
legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
{:ok, date} ->
if Timex.before?(date, @google_analytics_new_user_metric_date) do
redirect(conn,
to:
Routes.universal_analytics_path(conn, :user_metric_notice, site.domain,
view_id: view_id,
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
legacy: legacy
)
)
else
redirect(conn,
to:
Routes.universal_analytics_path(conn, :confirm, site.domain,
view_id: view_id,
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
legacy: legacy
)
)
end
end
end
def confirm(conn, %{
"view_id" => view_id,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
}) do
site = conn.assigns.site
start_date = Plausible.Google.UA.API.get_analytics_start_date(access_token, view_id)
end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)
{:ok, {view_name, view_id}} = Plausible.Google.UA.API.get_view(access_token, view_id)
conn
|> assign(:skip_plausible_tracking, true)
|> render("confirm.html",
access_token: access_token,
refresh_token: refresh_token,
expires_at: expires_at,
site: site,
selected_view_id: view_id,
selected_view_id_name: view_name,
start_date: start_date,
end_date: end_date,
legacy: legacy,
layout: {PlausibleWeb.LayoutView, "focus.html"}
)
end
def import(conn, %{
"view_id" => view_id,
"start_date" => start_date,
"end_date" => end_date,
"access_token" => access_token,
"refresh_token" => refresh_token,
"expires_at" => expires_at,
"legacy" => legacy
}) do
site = conn.assigns.site
current_user = conn.assigns.current_user
redirect_route =
if legacy == "true" do
Routes.site_path(conn, :settings_integrations, site.domain)
else
Routes.site_path(conn, :settings_imports_exports, site.domain)
end
{:ok, _} =
Plausible.Imported.UniversalAnalytics.new_import(
site,
current_user,
view_id: view_id,
start_date: start_date,
end_date: end_date,
access_token: access_token,
refresh_token: refresh_token,
token_expires_at: expires_at,
legacy: legacy == "true"
)
conn
|> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
|> redirect(external: redirect_route)
end
end

24
lib/plausible_web/router.ex
View File

@ -374,17 +374,27 @@ defmodule PlausibleWeb.Router do
delete "/:website/stats", SiteController, :reset_stats delete "/:website/stats", SiteController, :reset_stats
get "/:website/import/google-analytics/view-id", get "/:website/import/google-analytics/view-id",
SiteController, UniversalAnalyticsController,
:import_from_google_view_id_form :view_id_form
post "/:website/import/google-analytics/view-id", SiteController, :import_from_google_view_id post "/:website/import/google-analytics/view-id", UniversalAnalyticsController, :view_id
get "/:website/import/google-analytics/user-metric", get "/:website/import/google-analytics/user-metric",
SiteController, UniversalAnalyticsController,
:import_from_google_user_metric_notice :user_metric_notice
get "/:website/import/google-analytics/confirm", UniversalAnalyticsController, :confirm
post "/:website/settings/google-import", UniversalAnalyticsController, :import
get "/:website/import/google-analytics4/property",
GoogleAnalytics4Controller,
:property_form
post "/:website/import/google-analytics4/property", GoogleAnalytics4Controller, :property
get "/:website/import/google-analytics4/confirm", GoogleAnalytics4Controller, :confirm
post "/:website/settings/google4-import", GoogleAnalytics4Controller, :import
get "/:website/import/google-analytics/confirm", SiteController, :import_from_google_confirm
post "/:website/settings/google-import", SiteController, :import_from_google
delete "/:website/settings/forget-imported", SiteController, :forget_imported delete "/:website/settings/forget-imported", SiteController, :forget_imported
delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import

46
lib/plausible_web/templates/google_analytics4/confirm.html.heex Normal file
View File

@ -0,0 +1,46 @@
<%= form_for @conn, Routes.google_analytics4_path(@conn, :import, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
<h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
<%= hidden_input(f, :access_token, value: @access_token) %>
<%= hidden_input(f, :refresh_token, value: @refresh_token) %>
<%= hidden_input(f, :expires_at, value: @expires_at) %>
<%= case @start_date do %>
<% {:ok, start_date} -> %>
<div class="mt-6 text-sm text-gray-500 dark:text-gray-200">
Stats from this property and time period will be imported from your Google Analytics 4 account to your Plausible dashboard
</div>
<div class="mt-6">
<%= styled_label(f, :property, "Google Analytics 4 property") %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= @selected_property_name %>
</span>
<%= hidden_input(f, :property, readonly: "true", value: @selected_property) %>
</div>
<div class="flex justify-between mt-3">
<div class="w-36">
<%= styled_label(f, :start_date, "From") %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= PlausibleWeb.EmailView.date_format(start_date) %>
</span>
<%= hidden_input(f, :start_date, value: start_date, readonly: "true") %>
</div>
<div class="align-middle pt-4 dark:text-gray-100">&rarr;</div>
<div class="w-36">
<%= styled_label(f, :end_date, "To") %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= PlausibleWeb.EmailView.date_format(@end_date) %>
</span>
<%= hidden_input(f, :end_date, value: @end_date, readonly: "true") %>
</div>
</div>
<% {:error, error} -> %>
<p class="text-gray-700 dark:text-gray-300 mt-6">
The following error occurred when fetching your Google Analytics 4 data.
</p>
<p class="text-red-700 font-medium mt-3"><%= error %></p>
<% end %>
<%= submit("Confirm import", class: "button mt-6") %>
<% end %>

19
lib/plausible_web/templates/google_analytics4/property_form.html.heex Normal file
View File

@ -0,0 +1,19 @@
<%= form_for @conn, Routes.google_analytics4_path(@conn, :property, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
<h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics 4</h2>
<%= hidden_input(f, :access_token, value: @access_token) %>
<%= hidden_input(f, :refresh_token, value: @refresh_token) %>
<%= hidden_input(f, :expires_at, value: @expires_at) %>
<div class="mt-6 text-sm text-gray-500 dark:text-gray-200">
Choose the property in your Google Analytics 4 account that will be imported to the <%= @site.domain %> dashboard.
</div>
<div class="mt-3">
<%= styled_label(f, :property, "Google Analytics 4 property") %>
<%= styled_select(f, :property, @properties, prompt: "(Choose property)", required: "true") %>
<%= styled_error(@conn.assigns[:selected_property_error]) %>
</div>
<%= submit("Continue ->", class: "button mt-6") %>
<% end %>

26
lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex
View File

@ -1,26 +0,0 @@
<div class="max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8">
<h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
<div class="mt-6 text-sm text-gray-500 dark:text-gray-200">
<p>
<svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4 inline text-orange-600" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />
</svg>
Important: Since your GA property includes data from before 23rd August 2016, you have to take an extra step to make sure we can import data smoothly.
</p>
<ol class="mt-4">
<li>1. Navigate to the GA property you want to import from</li>
<li>2. Go to Admin &gt; Property Settings &gt; User Analysis</li>
<li>3. Make sure <i>Enable Users Metric in Reporting</i> is <b>OFF</b></li>
</ol>
<p class="mt-4">
The setting may take a few minutes to take effect. If your imported data is showing 0 visitors in unexpected places, it's probably caused by this and you
can try importing again later.
</p>
</div>
<%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token, refresh_token: @refresh_token, expires_at: @expires_at, legacy: @legacy), class: "button mt-6") %>
</div>

2
lib/plausible_web/templates/site/settings_google_import.html.heex
View File

@ -92,7 +92,7 @@
<% end %> <% end %>
<PlausibleWeb.Components.Google.button <PlausibleWeb.Components.Google.button
id="analytics-connect" id="analytics-connect"
to={Plausible.Google.Api.import_authorize_url(@site.id, "import", true)} to={Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: true)}
/> />
<% end %> <% end %>
<% else %> <% else %>

10
lib/plausible_web/templates/site/settings_imports_exports.html.heex
View File

@ -14,20 +14,22 @@
<PlausibleWeb.Components.Generic.button_link <PlausibleWeb.Components.Generic.button_link
class="w-36 h-20" class="w-36 h-20"
theme="bright" theme="bright"
href={Plausible.Google.Api.import_authorize_url(@site.id, "import", false)} href={Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: false)}
> >
<img src="/images/icon/universal_analytics_logo.svg" alt="New Universal Analytics import" /> <img src="/images/icon/universal_analytics_logo.svg" alt="New Universal Analytics import" />
</PlausibleWeb.Components.Generic.button_link> </PlausibleWeb.Components.Generic.button_link>
<PlausibleWeb.Components.Generic.button_link <PlausibleWeb.Components.Generic.button_link
class="w-36 h-20 opacity-40 cursor-not-allowed" class="w-36 h-20"
theme="bright" theme="bright"
href="" href={
Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: false, ga4: true)
}
> >
<img <img
src="/images/icon/google_analytics_4_logo.svg" src="/images/icon/google_analytics_4_logo.svg"
width="110" width="110"
alt="New Universal Analytics import" alt="New Google Analytics 4 import"
/> />
</PlausibleWeb.Components.Generic.button_link> </PlausibleWeb.Components.Generic.button_link>

2
lib/plausible_web/templates/site/settings_search_console.html.heex
View File

@ -82,7 +82,7 @@
<% else %> <% else %>
<PlausibleWeb.Components.Google.button <PlausibleWeb.Components.Google.button
id="search-console-connect" id="search-console-connect"
to={Plausible.Google.Api.search_console_authorize_url(@site.id, "search-console")} to={Plausible.Google.API.search_console_authorize_url(@site.id, "search-console")}
/> />
<div class="text-gray-700 dark:text-gray-300 mt-8"> <div class="text-gray-700 dark:text-gray-300 mt-8">
NB: You also need to set up your site on NB: You also need to set up your site on

33
lib/plausible_web/templates/site/import_from_google_confirm.html.eex → lib/plausible_web/templates/universal_analytics/confirm.html.heex
View File

@ -1,4 +1,4 @@
<%= form_for @conn, Routes.site_path(@conn, :import_from_google, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %> <%= form_for @conn, Routes.universal_analytics_path(@conn, :import, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
<h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2> <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
<%= hidden_input(f, :access_token, value: @access_token) %> <%= hidden_input(f, :access_token, value: @access_token) %>
@ -8,33 +8,40 @@
<%= case @start_date do %> <%= case @start_date do %>
<% {:ok, start_date} -> %> <% {:ok, start_date} -> %>
<div class="mt-6 text-sm text-gray-500 dark:text-gray-200"> <div class="mt-6 text-sm text-gray-500 dark:text-gray-200">
Stats from this property and time period will be imported from your Google Analytics account to your Plausible dashboard Stats from this property and time period will be imported from your Google Analytics account to your Plausible dashboard
</div> </div>
<div class="mt-6"> <div class="mt-6">
<%= styled_label(f, :view_id, "Google Analytics view") %> <%= styled_label(f, :view_id, "Google Analytics view") %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800"><%= @selected_view_id_name %></span> <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= hidden_input f, :view_id, readonly: "true", value: @selected_view_id %> <%= @selected_view_id_name %>
</span>
<%= hidden_input(f, :view_id, readonly: "true", value: @selected_view_id) %>
</div> </div>
<div class="flex justify-between mt-3"> <div class="flex justify-between mt-3">
<div class="w-36"> <div class="w-36">
<%= styled_label f, :start_date, "From" %> <%= styled_label(f, :start_date, "From") %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800"><%= PlausibleWeb.EmailView.date_format(start_date) %></span> <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= hidden_input f, :start_date, value: start_date, readonly: "true" %> <%= PlausibleWeb.EmailView.date_format(start_date) %>
</span>
<%= hidden_input(f, :start_date, value: start_date, readonly: "true") %>
</div> </div>
<div class="align-middle pt-4 dark:text-gray-100">&rarr;</div> <div class="align-middle pt-4 dark:text-gray-100">&rarr;</div>
<div class="w-36"> <div class="w-36">
<%= styled_label f, :end_date, "To" %> <%= styled_label(f, :end_date, "To") %>
<span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800"><%= PlausibleWeb.EmailView.date_format(@end_date) %></span> <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
<%= hidden_input f, :end_date, value: @end_date, readonly: "true" %> <%= PlausibleWeb.EmailView.date_format(@end_date) %>
</span>
<%= hidden_input(f, :end_date, value: @end_date, readonly: "true") %>
</div> </div>
</div> </div>
<% {:error, error} -> %> <% {:error, error} -> %>
<p class="text-gray-700 dark:text-gray-300 mt-6">The following error occurred when fetching your Google Analytics data.</p> <p class="text-gray-700 dark:text-gray-300 mt-6">
The following error occurred when fetching your Google Analytics data.
</p>
<p class="text-red-700 font-medium mt-3"><%= error %></p> <p class="text-red-700 font-medium mt-3"><%= error %></p>
<% end %> <% end %>
<%= submit "Confirm import", class: "button mt-6" %> <%= submit("Confirm import", class: "button mt-6") %>
<% end %> <% end %>

46
lib/plausible_web/templates/universal_analytics/user_metric_form.html.heex Normal file
View File

@ -0,0 +1,46 @@
<div class="max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8">
<h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
<div class="mt-6 text-sm text-gray-500 dark:text-gray-200">
<p>
<svg
xmlns="http://www.w3.org/2000/svg"
class="h-4 w-4 inline text-orange-600"
fill="none"
viewBox="0 0 24 24"
stroke="currentColor"
stroke-width="2"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z"
/>
</svg>
Important: Since your GA property includes data from before 23rd August 2016, you have to take an extra step to make sure we can import data smoothly.
</p>
<ol class="mt-4">
<li>1. Navigate to the GA property you want to import from</li>
<li>2. Go to Admin &gt; Property Settings &gt; User Analysis</li>
<li>3. Make sure <i>Enable Users Metric in Reporting</i> is <b>OFF</b></li>
</ol>
<p class="mt-4">
The setting may take a few minutes to take effect. If your imported data is showing 0 visitors in unexpected places, it's probably caused by this and you
can try importing again later.
</p>
</div>
<%= link("Continue ->",
to:
Routes.universal_analytics_path(@conn, :confirm, @site.domain,
view_id: @view_id,
access_token: @access_token,
refresh_token: @refresh_token,
expires_at: @expires_at,
legacy: @legacy
),
class: "button mt-6"
) %>
</div>

6
lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex → lib/plausible_web/templates/universal_analytics/view_id_form.html.heex
View File

@ -1,4 +1,4 @@
<%= form_for @conn, Routes.site_path(@conn, :import_from_google_view_id, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %> <%= form_for @conn, Routes.universal_analytics_path(@conn, :view_id, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
<h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2> <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
<%= hidden_input(f, :access_token, value: @access_token) %> <%= hidden_input(f, :access_token, value: @access_token) %>
@ -12,9 +12,9 @@
<div class="mt-3"> <div class="mt-3">
<%= styled_label(f, :view_id, "Google Analytics view") %> <%= styled_label(f, :view_id, "Google Analytics view") %>
<%= styled_select f, :view_id, @view_ids, prompt: "(Choose view)", required: "true" %> <%= styled_select(f, :view_id, @view_ids, prompt: "(Choose view)", required: "true") %>
<%= styled_error(@conn.assigns[:selected_view_id_error]) %> <%= styled_error(@conn.assigns[:selected_view_id_error]) %>
</div> </div>
<%= submit "Continue ->", class: "button mt-6" %> <%= submit("Continue ->", class: "button mt-6") %>
<% end %> <% end %>

4
lib/plausible_web/views/google_analytics4_view.ex Normal file
View File

@ -0,0 +1,4 @@
defmodule PlausibleWeb.GoogleAnalytics4View do
use PlausibleWeb, :view
use Plausible
end

4
lib/plausible_web/views/universal_analytics_view.ex Normal file
View File

@ -0,0 +1,4 @@
defmodule PlausibleWeb.UniversalAnalyticsView do
use PlausibleWeb, :view
use Plausible
end

44
test/plausible/google/api_test.exs
View File

@ -1,8 +1,8 @@
defmodule Plausible.Google.ApiTest do defmodule Plausible.Google.APITest do
use Plausible.DataCase, async: true use Plausible.DataCase, async: true
use Plausible.Test.Support.HTTPMocker use Plausible.Test.Support.HTTPMocker
alias Plausible.Google.Api alias Plausible.Google
alias Plausible.Imported.UniversalAnalytics alias Plausible.Imported.UniversalAnalytics
import ExUnit.CaptureLog import ExUnit.CaptureLog
@ -44,7 +44,7 @@ defmodule Plausible.Google.ApiTest do
Plausible.Imported.Buffer.insert_many(buffer, table, records) Plausible.Imported.Buffer.insert_many(buffer, table, records)
end end
assert :ok == Plausible.Google.Api.import_analytics(date_range, view_id, auth, persist_fn) assert :ok == Google.UA.API.import_analytics(date_range, view_id, auth, persist_fn)
Plausible.Imported.Buffer.flush(buffer) Plausible.Imported.Buffer.flush(buffer)
Plausible.Imported.Buffer.stop(buffer) Plausible.Imported.Buffer.stop(buffer)
@ -86,7 +86,7 @@ defmodule Plausible.Google.ApiTest do
Plausible.Imported.Buffer.insert_many(buffer, table, records) Plausible.Imported.Buffer.insert_many(buffer, table, records)
end end
assert :ok == Plausible.Google.Api.import_analytics(range, "123551", auth, persist_fn) assert :ok == Google.UA.API.import_analytics(range, "123551", auth, persist_fn)
Plausible.Imported.Buffer.flush(buffer) Plausible.Imported.Buffer.flush(buffer)
Plausible.Imported.Buffer.stop(buffer) Plausible.Imported.Buffer.stop(buffer)
@ -98,7 +98,7 @@ defmodule Plausible.Google.ApiTest do
@tag :slow @tag :slow
test "will fetch and persist import data from Google Analytics" do test "will fetch and persist import data from Google Analytics" do
request = %Plausible.Google.ReportRequest{ request = %Plausible.Google.UA.ReportRequest{
dataset: "imported_exit_pages", dataset: "imported_exit_pages",
view_id: "123", view_id: "123",
date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]), date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]),
@ -127,7 +127,7 @@ defmodule Plausible.Google.ApiTest do
hideValueRanges: true, hideValueRanges: true,
metrics: [%{expression: "ga:users"}, %{expression: "ga:exits"}], metrics: [%{expression: "ga:users"}, %{expression: "ga:exits"}],
orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}], orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],
pageSize: 10000, pageSize: 10_000,
pageToken: nil, pageToken: nil,
viewId: "123" viewId: "123"
} }
@ -139,7 +139,7 @@ defmodule Plausible.Google.ApiTest do
) )
assert :ok = assert :ok =
Api.fetch_and_persist(request, Google.UA.API.fetch_and_persist(request,
sleep_time: 0, sleep_time: 0,
persist_fn: fn dataset, row -> persist_fn: fn dataset, row ->
assert dataset == "imported_exit_pages" assert dataset == "imported_exit_pages"
@ -167,7 +167,7 @@ defmodule Plausible.Google.ApiTest do
end end
) )
request = %Plausible.Google.ReportRequest{ request = %Plausible.Google.UA.ReportRequest{
view_id: "123", view_id: "123",
date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]), date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]),
dimensions: ["ga:date"], dimensions: ["ga:date"],
@ -178,7 +178,7 @@ defmodule Plausible.Google.ApiTest do
} }
assert {:error, :request_failed} = assert {:error, :request_failed} =
Api.fetch_and_persist(request, Google.UA.API.fetch_and_persist(request,
sleep_time: 0, sleep_time: 0,
persist_fn: fn _dataset, _rows -> :ok end persist_fn: fn _dataset, _rows -> :ok end
) )
@ -197,7 +197,7 @@ defmodule Plausible.Google.ApiTest do
end end
) )
request = %Plausible.Google.ReportRequest{ request = %Plausible.Google.UA.ReportRequest{
dataset: "imported_exit_pages", dataset: "imported_exit_pages",
view_id: "123", view_id: "123",
date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]), date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]),
@ -209,7 +209,7 @@ defmodule Plausible.Google.ApiTest do
} }
assert :ok == assert :ok ==
Api.fetch_and_persist(request, Google.UA.API.fetch_and_persist(request,
sleep_time: 0, sleep_time: 0,
persist_fn: fn dataset, rows -> persist_fn: fn dataset, rows ->
assert dataset == "imported_exit_pages" assert dataset == "imported_exit_pages"
@ -253,7 +253,7 @@ defmodule Plausible.Google.ApiTest do
query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
assert {:error, "google_auth_error"} = Plausible.Google.Api.fetch_stats(site, query, 5) assert {:error, "google_auth_error"} = Google.API.fetch_stats(site, query, 5)
end end
test "returns whatever error code google returns on API client error", %{site: site} do test "returns whatever error code google returns on API client error", %{site: site} do
@ -270,7 +270,7 @@ defmodule Plausible.Google.ApiTest do
query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
assert {:error, "some_error"} = Plausible.Google.Api.fetch_stats(site, query, 5) assert {:error, "some_error"} = Google.API.fetch_stats(site, query, 5)
end end
test "returns generic HTTP error and logs it", %{site: site} do test "returns generic HTTP error and logs it", %{site: site} do
@ -290,7 +290,7 @@ defmodule Plausible.Google.ApiTest do
log = log =
capture_log(fn -> capture_log(fn ->
assert {:error, "failed_to_list_stats"} = assert {:error, "failed_to_list_stats"} =
Plausible.Google.Api.fetch_stats(site, query, 5) Google.API.fetch_stats(site, query, 5)
end) end)
assert log =~ "Google Analytics: failed to list stats: %Finch.Error{reason: :some_reason}" assert log =~ "Google Analytics: failed to list stats: %Finch.Error{reason: :some_reason}"
@ -314,7 +314,7 @@ defmodule Plausible.Google.ApiTest do
[ [
%{name: ["keyword1", "keyword2"], visitors: 25}, %{name: ["keyword1", "keyword2"], visitors: 25},
%{name: ["keyword3", "keyword4"], visitors: 15} %{name: ["keyword3", "keyword4"], visitors: 15}
]} = Plausible.Google.Api.fetch_stats(site, query, 5) ]} = Google.API.fetch_stats(site, query, 5)
end end
test "returns next page when page argument is set", %{user: user, site: site} do test "returns next page when page argument is set", %{user: user, site: site} do
@ -336,7 +336,7 @@ defmodule Plausible.Google.ApiTest do
[ [
%{name: ["keyword1", "keyword2"], visitors: 25}, %{name: ["keyword1", "keyword2"], visitors: 25},
%{name: ["keyword3", "keyword4"], visitors: 15} %{name: ["keyword3", "keyword4"], visitors: 15}
]} = Plausible.Google.Api.fetch_stats(site, query, 5) ]} = Google.API.fetch_stats(site, query, 5)
end end
test "defaults first page when page argument is not set", %{user: user, site: site} do test "defaults first page when page argument is not set", %{user: user, site: site} do
@ -355,7 +355,7 @@ defmodule Plausible.Google.ApiTest do
[ [
%{name: ["keyword1", "keyword2"], visitors: 25}, %{name: ["keyword1", "keyword2"], visitors: 25},
%{name: ["keyword3", "keyword4"], visitors: 15} %{name: ["keyword3", "keyword4"], visitors: 15}
]} = Plausible.Google.Api.fetch_stats(site, query, 5) ]} = Google.API.fetch_stats(site, query, 5)
end end
test "returns error when token refresh fails", %{user: user, site: site} do test "returns error when token refresh fails", %{user: user, site: site} do
@ -372,7 +372,7 @@ defmodule Plausible.Google.ApiTest do
query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
assert {:error, "invalid_grant"} = Plausible.Google.Api.fetch_stats(site, query, 5) assert {:error, "invalid_grant"} = Google.API.fetch_stats(site, query, 5)
end end
end end
@ -393,7 +393,7 @@ defmodule Plausible.Google.ApiTest do
%{ %{
"one.test" => [{"57238190 - one.test", "57238190"}], "one.test" => [{"57238190 - one.test", "57238190"}],
"two.test" => [{"54460083 - two.test", "54460083"}] "two.test" => [{"54460083 - two.test", "54460083"}]
}} == Plausible.Google.Api.list_views("access_token") }} == Google.UA.API.list_views("access_token")
end end
test "list_views/1 returns authentication_failed when request fails with HTTP 403" do test "list_views/1 returns authentication_failed when request fails with HTTP 403" do
@ -405,7 +405,7 @@ defmodule Plausible.Google.ApiTest do
end end
) )
assert {:error, :authentication_failed} == Plausible.Google.Api.list_views("access_token") assert {:error, :authentication_failed} == Google.UA.API.list_views("access_token")
end end
test "list_views/1 returns authentication_failed when request fails with HTTP 401" do test "list_views/1 returns authentication_failed when request fails with HTTP 401" do
@ -417,7 +417,7 @@ defmodule Plausible.Google.ApiTest do
end end
) )
assert {:error, :authentication_failed} == Plausible.Google.Api.list_views("access_token") assert {:error, :authentication_failed} == Google.UA.API.list_views("access_token")
end end
test "list_views/1 returns error when request fails with HTTP 500" do test "list_views/1 returns error when request fails with HTTP 500" do
@ -429,6 +429,6 @@ defmodule Plausible.Google.ApiTest do
end end
) )
assert {:error, :unknown} == Plausible.Google.Api.list_views("access_token") assert {:error, :unknown} == Google.UA.API.list_views("access_token")
end end
end end

57
test/plausible/google/ga4/api_test.exs Normal file
View File

@ -0,0 +1,57 @@
defmodule Plausible.Google.GA4.APITest do
use Plausible.DataCase, async: true
import Mox
alias Plausible.Google.GA4
setup :verify_on_exit!
describe "list_properties/1" do
test "returns list of properties grouped by accounts" do
result = Jason.decode!(File.read!("fixture/ga4_list_properties.json"))
expect(Plausible.HTTPClient.Mock, :get, fn _url, _opts ->
{:ok, %Finch.Response{status: 200, body: result}}
end)
assert {:ok, accounts} = GA4.API.list_properties("some_access_token")
assert [
{"account.one (accounts/28425178)",
[{"account.one - GA4 (properties/428685906)", "properties/428685906"}]},
{"Demo Account (accounts/54516992)",
[
{"GA4 - Flood-It! (properties/153293282)", "properties/153293282"},
{"GA4 - Google Merch Shop (properties/213025502)", "properties/213025502"}
]}
] = accounts
end
end
describe "get_property/2" do
test "returns tuple consisting of display name and value of a property" do
result = Jason.decode!(File.read!("fixture/ga4_list_properties.json"))
expect(Plausible.HTTPClient.Mock, :get, fn _url, _opts ->
{:ok, %Finch.Response{status: 200, body: result}}
end)
assert {:ok, {"GA4 - Flood-It! (properties/153293282)", "properties/153293282"}} =
GA4.API.get_property("some_access_token", "properties/153293282")
end
end
describe "get_analytics_start_date/2" do
test "returns stats start date for a given property" do
result = Jason.decode!(File.read!("fixture/ga4_start_date.json"))
expect(Plausible.HTTPClient.Mock, :post, fn _url, _headers, _body ->
{:ok, %Finch.Response{status: 200, body: result}}
end)
assert {:ok, ~D[2024-02-22]} =
GA4.API.get_analytics_start_date("some_access_token", "properties/153293282")
end
end
end

98
test/plausible/imported/google_analytics4_test.exs Normal file
View File

@ -0,0 +1,98 @@
defmodule Plausible.Imported.GoogleAnalytics4Test do
use Plausible.DataCase, async: true
import Mox
import Ecto.Query, only: [from: 2]
alias Plausible.Imported.GoogleAnalytics4
@refresh_token_body Jason.decode!(File.read!("fixture/ga_refresh_token.json"))
@full_report_mock [
"fixture/ga4_report_imported_visitors.json",
"fixture/ga4_report_imported_sources.json",
"fixture/ga4_report_imported_pages.json",
"fixture/ga4_report_imported_entry_pages.json",
"fixture/ga4_report_imported_locations.json",
"fixture/ga4_report_imported_devices.json",
"fixture/ga4_report_imported_browsers.json",
"fixture/ga4_report_imported_operating_systems.json"
]
|> Enum.map(&File.read!/1)
|> Enum.map(&Jason.decode!/1)
setup :verify_on_exit!
describe "parse_args/1 and import_data/2" do
setup [:create_user, :create_new_site]
test "imports data returned from GA4 Data API", %{user: user, site: site} do
past = DateTime.add(DateTime.utc_now(), -3600, :second)
{:ok, job} =
Plausible.Imported.GoogleAnalytics4.new_import(
site,
user,
property: "properties/123456",
start_date: ~D[2024-02-20],
end_date: Date.utc_today(),
access_token: "redacted_access_token",
refresh_token: "redacted_refresh_token",
token_expires_at: DateTime.to_iso8601(past)
)
site_import = Plausible.Imported.get_import(job.args.import_id)
opts = job |> Repo.reload!() |> Map.get(:args) |> GoogleAnalytics4.parse_args()
opts = Keyword.put(opts, :flush_interval_ms, 10)
expect(Plausible.HTTPClient.Mock, :post, fn "https://www.googleapis.com/oauth2/v4/token",
headers,
body ->
assert [{"content-type", "application/x-www-form-urlencoded"}] == headers
assert %{
grant_type: :refresh_token,
redirect_uri: "http://localhost:8000/auth/google/callback",
refresh_token: "redacted_refresh_token"
} = body
{:ok, %Finch.Response{status: 200, body: @refresh_token_body}}
end)
for report <- @full_report_mock do
expect(Plausible.HTTPClient.Mock, :post, fn _url, headers, _body, _opts ->
assert [{"Authorization", "Bearer 1/fFAGRNJru1FTz70BzhT3Zg"}] == headers
{:ok, %Finch.Response{status: 200, body: report}}
end)
end
Enum.each(Plausible.Imported.tables(), fn table ->
query = from(imported in table, where: imported.site_id == ^site.id)
assert await_clickhouse_count(query, 0)
end)
assert :ok = GoogleAnalytics4.import_data(site_import, opts)
Enum.each(Plausible.Imported.tables(), fn table ->
count =
case table do
"imported_sources" -> 3
"imported_visitors" -> 3
"imported_pages" -> 8
"imported_entry_pages" -> 4
"imported_exit_pages" -> 0
"imported_locations" -> 4
"imported_devices" -> 4
"imported_browsers" -> 5
"imported_operating_systems" -> 4
end
query = from(imported in table, where: imported.site_id == ^site.id)
assert await_clickhouse_count(query, count)
end)
end
end
end

2
test/plausible_web/controllers/api/stats_controller/sources_test.exs
View File

@ -1461,7 +1461,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
]) ])
conn = get(conn, "/api/stats/#{site.domain}/referrers/Google?period=day") conn = get(conn, "/api/stats/#{site.domain}/referrers/Google?period=day")
{:ok, terms} = Plausible.Google.Api.Mock.fetch_stats(nil, nil, nil) {:ok, terms} = Plausible.Google.API.Mock.fetch_stats(nil, nil, nil)
assert json_response(conn, 200) == %{ assert json_response(conn, 200) == %{
"total_visitors" => 2, "total_visitors" => 2,

6
test/support/google_api_mock.ex
View File

@ -1,4 +1,8 @@
defmodule Plausible.Google.Api.Mock do defmodule Plausible.Google.API.Mock do
@moduledoc """
Mock of API to Google services.
"""
def fetch_stats(_auth, _query, _limit) do def fetch_stats(_auth, _query, _limit) do
{:ok, {:ok,
[ [