From 4d7d88cfeccbe35224bad5ffd3ed161784fa67d7 Mon Sep 17 00:00:00 2001 From: Adrian Gruntkowski Date: Tue, 12 Mar 2024 18:08:25 +0100 Subject: [PATCH] Implement basics of GA4 import (#3851) * Implement LV date input using flatpickr * Implement basics of GA4 import (very dirty WIP) * Split Google HTTP API into UA and GA4 specific parts * Add a quick way to record GA4 API responses * Add first GA4 import fixtures with GA4 Data API responses * Extract GA4 and UA specific logic form Google API * Extract UA and GA4 specific actions to distinct controllers * Add integration test for GA4 importer * Update GA4 fixtures * Test GA4 API * Add debug logging and fix paginating through API results in in GA4 import * Revert "Implement LV date input using flatpickr" This reverts commit c696f8ee39d5702f27015c09a4f079ca124cc7bb. * Fix note --- config/config.exs | 2 +- config/test.exs | 2 +- fixture/ga4_list_properties.json | 41 +++ fixture/ga4_report_imported_browsers.json | 161 +++++++++++ fixture/ga4_report_imported_devices.json | 137 ++++++++++ fixture/ga4_report_imported_entry_pages.json | 137 ++++++++++ fixture/ga4_report_imported_locations.json | 167 ++++++++++++ ...ga4_report_imported_operating_systems.json | 137 ++++++++++ fixture/ga4_report_imported_pages.json | 232 ++++++++++++++++ fixture/ga4_report_imported_sources.json | 161 +++++++++++ fixture/ga4_report_imported_visitors.json | 114 ++++++++ fixture/ga4_start_date.json | 38 +++ lib/plausible/google/api.ex | 160 ++--------- lib/plausible/google/ga4/api.ex | 142 ++++++++++ lib/plausible/google/ga4/http.ex | 199 ++++++++++++++ lib/plausible/google/ga4/report_request.ex | 122 +++++++++ lib/plausible/google/http.ex | 159 +---------- lib/plausible/google/ua/api.ex | 146 ++++++++++ lib/plausible/google/ua/http.ex | 167 ++++++++++++ .../google/{ => ua}/report_request.ex | 6 +- lib/plausible/imported/buffer.ex | 17 +- lib/plausible/imported/google_analytics4.ex | 253 ++++++++++++++++++ lib/plausible/imported/import_sources.ex | 1 + lib/plausible/imported/universal_analytics.ex | 2 +- .../controllers/auth_controller.ex | 49 ++-- .../google_analytics4_controller.ex | 143 ++++++++++ .../controllers/site_controller.ex | 194 +------------- .../universal_analytics_controller.ex | 198 ++++++++++++++ lib/plausible_web/router.ex | 24 +- .../google_analytics4/confirm.html.heex | 46 ++++ .../google_analytics4/property_form.html.heex | 19 ++ ...port_from_google_user_metric_form.html.eex | 26 -- .../site/settings_google_import.html.heex | 2 +- .../site/settings_imports_exports.html.heex | 10 +- .../site/settings_search_console.html.heex | 2 +- .../confirm.html.heex} | 33 ++- .../user_metric_form.html.heex | 46 ++++ .../view_id_form.html.heex} | 6 +- .../views/google_analytics4_view.ex | 4 + .../views/universal_analytics_view.ex | 4 + test/plausible/google/api_test.exs | 44 +-- test/plausible/google/ga4/api_test.exs | 57 ++++ .../imported/google_analytics4_test.exs | 98 +++++++ .../api/stats_controller/sources_test.exs | 2 +- test/support/google_api_mock.ex | 6 +- 45 files changed, 3118 insertions(+), 598 deletions(-) create mode 100644 fixture/ga4_list_properties.json create mode 100644 fixture/ga4_report_imported_browsers.json create mode 100644 fixture/ga4_report_imported_devices.json create mode 100644 fixture/ga4_report_imported_entry_pages.json create mode 100644 fixture/ga4_report_imported_locations.json create mode 100644 fixture/ga4_report_imported_operating_systems.json create mode 100644 fixture/ga4_report_imported_pages.json create mode 100644 fixture/ga4_report_imported_sources.json create mode 100644 fixture/ga4_report_imported_visitors.json create mode 100644 fixture/ga4_start_date.json create mode 100644 lib/plausible/google/ga4/api.ex create mode 100644 lib/plausible/google/ga4/http.ex create mode 100644 lib/plausible/google/ga4/report_request.ex create mode 100644 lib/plausible/google/ua/api.ex create mode 100644 lib/plausible/google/ua/http.ex rename lib/plausible/google/{ => ua}/report_request.ex (95%) create mode 100644 lib/plausible/imported/google_analytics4.ex create mode 100644 lib/plausible_web/controllers/google_analytics4_controller.ex create mode 100644 lib/plausible_web/controllers/universal_analytics_controller.ex create mode 100644 lib/plausible_web/templates/google_analytics4/confirm.html.heex create mode 100644 lib/plausible_web/templates/google_analytics4/property_form.html.heex delete mode 100644 lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex rename lib/plausible_web/templates/{site/import_from_google_confirm.html.eex => universal_analytics/confirm.html.heex} (52%) create mode 100644 lib/plausible_web/templates/universal_analytics/user_metric_form.html.heex rename lib/plausible_web/templates/{site/import_from_google_view_id_form.html.eex => universal_analytics/view_id_form.html.heex} (66%) create mode 100644 lib/plausible_web/views/google_analytics4_view.ex create mode 100644 lib/plausible_web/views/universal_analytics_view.ex create mode 100644 test/plausible/google/ga4/api_test.exs create mode 100644 test/plausible/imported/google_analytics4_test.exs diff --git a/config/config.exs b/config/config.exs index b3a1f05e88..f7f2f32d80 100644 --- a/config/config.exs +++ b/config/config.exs @@ -45,7 +45,7 @@ config :ref_inspector, config :plausible, paddle_api: Plausible.Billing.PaddleApi, - google_api: Plausible.Google.Api + google_api: Plausible.Google.API config :plausible, # 30 minutes diff --git a/config/test.exs b/config/test.exs index 31182b2846..d1d9edfac0 100644 --- a/config/test.exs +++ b/config/test.exs @@ -16,7 +16,7 @@ config :plausible, Plausible.Mailer, adapter: Bamboo.TestAdapter config :plausible, paddle_api: Plausible.PaddleApi.Mock, - google_api: Plausible.Google.Api.Mock + google_api: Plausible.Google.API.Mock config :bamboo, :refute_timeout, 10 diff --git a/fixture/ga4_list_properties.json b/fixture/ga4_list_properties.json new file mode 100644 index 0000000000..f8aac0b1d6 --- /dev/null +++ b/fixture/ga4_list_properties.json @@ -0,0 +1,41 @@ +{ + "accountSummaries": [ + { + "account": "accounts/28425178", + "displayName": "account.one", + "name": "accountSummaries/28425178", + "propertySummaries": [ + { + "displayName": "account.one - GA4", + "parent": "accounts/28425178", + "property": "properties/428685906", + "propertyType": "PROPERTY_TYPE_ORDINARY" + } + ] + }, + { + "account": "accounts/45336102", + "displayName": "account.two", + "name": "accountSummaries/45336102" + }, + { + "account": "accounts/54516992", + "displayName": "Demo Account", + "name": "accountSummaries/54516992", + "propertySummaries": [ + { + "displayName": "GA4 - Flood-It!", + "parent": "accounts/54516992", + "property": "properties/153293282", + "propertyType": "PROPERTY_TYPE_ORDINARY" + }, + { + "displayName": "GA4 - Google Merch Shop", + "parent": "accounts/54516992", + "property": "properties/213025502", + "propertyType": "PROPERTY_TYPE_ORDINARY" + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_browsers.json b/fixture/ga4_report_imported_browsers.json new file mode 100644 index 0000000000..214ac2b268 --- /dev/null +++ b/fixture/ga4_report_imported_browsers.json @@ -0,0 +1,161 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "browser" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "sessions", + "type": "TYPE_INTEGER" + }, + { + "name": "bounces", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + } + ], + "rowCount": 5, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + }, + { + "value": "Safari" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "Chrome" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "Chrome" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + }, + { + "value": "4" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "Firefox" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "2" + }, + { + "value": "0" + }, + { + "value": "21" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "Safari" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + }, + { + "value": "4" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_devices.json b/fixture/ga4_report_imported_devices.json new file mode 100644 index 0000000000..e8bcc01047 --- /dev/null +++ b/fixture/ga4_report_imported_devices.json @@ -0,0 +1,137 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "deviceCategory" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "sessions", + "type": "TYPE_INTEGER" + }, + { + "name": "bounces", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + } + ], + "rowCount": 4, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + }, + { + "value": "mobile" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "desktop" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "desktop" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "3" + }, + { + "value": "0" + }, + { + "value": "25" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "mobile" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + }, + { + "value": "4" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_entry_pages.json b/fixture/ga4_report_imported_entry_pages.json new file mode 100644 index 0000000000..77370bb30f --- /dev/null +++ b/fixture/ga4_report_imported_entry_pages.json @@ -0,0 +1,137 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "landingPage" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "sessions", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + }, + { + "name": "bounces", + "type": "TYPE_INTEGER" + } + ], + "rowCount": 4, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + }, + { + "value": "/blog/firstpost" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "/" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "0" + }, + { + "value": "2" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "/" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "3" + }, + { + "value": "25" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "/blog/unicode-in-elixir" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "4" + }, + { + "value": "0" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_locations.json b/fixture/ga4_report_imported_locations.json new file mode 100644 index 0000000000..b731a21025 --- /dev/null +++ b/fixture/ga4_report_imported_locations.json @@ -0,0 +1,167 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "countryId" + }, + { + "name": "region" + }, + { + "name": "city" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "sessions", + "type": "TYPE_INTEGER" + }, + { + "name": "bounces", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + } + ], + "rowCount": 4, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + }, + { + "value": "PL" + }, + { + "value": "Masovian Voivodeship" + }, + { + "value": "Warsaw" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "US" + }, + { + "value": "California" + }, + { + "value": "(not set)" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "PL" + }, + { + "value": "Pomeranian Voivodeship" + }, + { + "value": "Gdansk" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "3" + }, + { + "value": "0" + }, + { + "value": "25" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "ES" + }, + { + "value": "Catalonia" + }, + { + "value": "Barcelona" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + }, + { + "value": "4" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_operating_systems.json b/fixture/ga4_report_imported_operating_systems.json new file mode 100644 index 0000000000..158fe1efa9 --- /dev/null +++ b/fixture/ga4_report_imported_operating_systems.json @@ -0,0 +1,137 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "operatingSystem" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "sessions", + "type": "TYPE_INTEGER" + }, + { + "name": "bounces", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + } + ], + "rowCount": 4, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + }, + { + "value": "iOS" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "Windows" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "Macintosh" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "3" + }, + { + "value": "0" + }, + { + "value": "25" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "iOS" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + }, + { + "value": "4" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_pages.json b/fixture/ga4_report_imported_pages.json new file mode 100644 index 0000000000..f7be04f265 --- /dev/null +++ b/fixture/ga4_report_imported_pages.json @@ -0,0 +1,232 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "hostName" + }, + { + "name": "pagePath" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "screenPageViews", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + } + ], + "rowCount": 8, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + }, + { + "value": "drawer.todo.computer" + }, + { + "value": "/blog/firstpost/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "drawer-4l3.pages.dev" + }, + { + "value": "/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "drawer.todo.computer" + }, + { + "value": "/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "drawer.todo.computer" + }, + { + "value": "/" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "3" + }, + { + "value": "7" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "drawer.todo.computer" + }, + { + "value": "/blog/unicode-in-elixir/" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "7" + }, + { + "value": "21" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "drawer.todo.computer" + }, + { + "value": "/about/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "drawer.todo.computer" + }, + { + "value": "/blog/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "drawer.todo.computer" + }, + { + "value": "/blog/firstpost/" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_sources.json b/fixture/ga4_report_imported_sources.json new file mode 100644 index 0000000000..4c5edb5179 --- /dev/null +++ b/fixture/ga4_report_imported_sources.json @@ -0,0 +1,161 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + }, + { + "name": "sessionSource" + }, + { + "name": "sessionMedium" + }, + { + "name": "sessionCampaignName" + }, + { + "name": "sessionManualAdContent" + }, + { + "name": "sessionGoogleAdsKeyword" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "sessions", + "type": "TYPE_INTEGER" + }, + { + "name": "bounces", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + } + ], + "rowCount": 3, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + }, + { + "value": "(direct)" + }, + { + "value": "(none)" + }, + { + "value": "(direct)" + }, + { + "value": "(not set)" + }, + { + "value": "(not set)" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + }, + { + "value": "(direct)" + }, + { + "value": "(none)" + }, + { + "value": "(direct)" + }, + { + "value": "(not set)" + }, + { + "value": "(not set)" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + }, + { + "value": "(direct)" + }, + { + "value": "(none)" + }, + { + "value": "(direct)" + }, + { + "value": "(not set)" + }, + { + "value": "(not set)" + } + ], + "metricValues": [ + { + "value": "3" + }, + { + "value": "4" + }, + { + "value": "0" + }, + { + "value": "29" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_report_imported_visitors.json b/fixture/ga4_report_imported_visitors.json new file mode 100644 index 0000000000..78d29b5e70 --- /dev/null +++ b/fixture/ga4_report_imported_visitors.json @@ -0,0 +1,114 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "totalUsers", + "type": "TYPE_INTEGER" + }, + { + "name": "screenPageViews", + "type": "TYPE_INTEGER" + }, + { + "name": "bounces", + "type": "TYPE_INTEGER" + }, + { + "name": "sessions", + "type": "TYPE_INTEGER" + }, + { + "name": "userEngagementDuration", + "type": "TYPE_SECONDS" + } + ], + "rowCount": 3, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240226" + } + ], + "metricValues": [ + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "1" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240224" + } + ], + "metricValues": [ + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "2" + }, + { + "value": "0" + } + ] + }, + { + "dimensionValues": [ + { + "value": "20240222" + } + ], + "metricValues": [ + { + "value": "3" + }, + { + "value": "13" + }, + { + "value": "0" + }, + { + "value": "4" + }, + { + "value": "29" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/fixture/ga4_start_date.json b/fixture/ga4_start_date.json new file mode 100644 index 0000000000..6c1d410d23 --- /dev/null +++ b/fixture/ga4_start_date.json @@ -0,0 +1,38 @@ +{ + "kind": "analyticsData#batchRunReports", + "reports": [ + { + "dimensionHeaders": [ + { + "name": "date" + } + ], + "kind": "analyticsData#runReport", + "metadata": { + "currencyCode": "USD", + "timeZone": "Europe/Warsaw" + }, + "metricHeaders": [ + { + "name": "screenPageViews", + "type": "TYPE_INTEGER" + } + ], + "rowCount": 3, + "rows": [ + { + "dimensionValues": [ + { + "value": "20240222" + } + ], + "metricValues": [ + { + "value": "13" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 085513886d..6b1e44b4ea 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -1,9 +1,13 @@ -defmodule Plausible.Google.Api do - alias Plausible.Google.{ReportRequest, HTTP} - use Timex - require Logger +defmodule Plausible.Google.API do + @moduledoc """ + API to Google services. + """ - @type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()} + use Timex + + alias Plausible.Google.HTTP + + require Logger @search_console_scope URI.encode_www_form( "email https://www.googleapis.com/auth/webmasters.readonly" @@ -17,9 +21,16 @@ defmodule Plausible.Google.Api do Jason.encode!([site_id, redirect_to]) end - def import_authorize_url(site_id, redirect_to, legacy \\ true) do + def import_authorize_url(site_id, redirect_to, opts \\ []) do + legacy = Keyword.get(opts, :legacy, true) + ga4 = Keyword.get(opts, :ga4, false) + "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <> - Jason.encode!([site_id, redirect_to, legacy]) + Jason.encode!([site_id, redirect_to, legacy, ga4]) + end + + def fetch_access_token!(code) do + HTTP.fetch_access_token!(code) end def fetch_verified_properties(auth) do @@ -53,138 +64,7 @@ defmodule Plausible.Google.Api do end end - @spec list_views(access_token :: String.t()) :: - {:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()} - @doc """ - Lists Google Analytics views grouped by hostname. - """ - def list_views(access_token) do - case HTTP.list_views_for_user(access_token) do - {:ok, %{"items" => views}} -> - views = Enum.group_by(views, &view_hostname/1, &view_names/1) - {:ok, views} - - error -> - error - end - end - - defp view_hostname(view) do - case view do - %{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host) - _any -> "Others" - end - end - - defp view_names(%{"name" => name, "id" => id}) do - {"#{id} - #{name}", id} - end - - @spec get_view(access_token :: String.t(), lookup_id :: String.t()) :: - {:ok, google_analytics_view()} | {:ok, nil} | {:error, term()} - @doc """ - Returns a single Google Analytics view if the user has access to it. - """ - def get_view(access_token, lookup_id) do - case list_views(access_token) do - {:ok, views} -> - view = - views - |> Map.values() - |> List.flatten() - |> Enum.find(fn {_name, id} -> id == lookup_id end) - - {:ok, view} - - {:error, cause} -> - {:error, cause} - end - end - - @type import_auth :: { - access_token :: String.t(), - refresh_token :: String.t(), - expires_at :: String.t() - } - - @per_page 7_500 - @backoff_factor :timer.seconds(10) - @max_attempts 5 - @spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) :: - :ok | {:error, term()} - @doc """ - Imports stats from a Google Analytics UA view to a Plausible site. - - This function fetches Google Analytics reports in batches of #{@per_page} per - request. The batches are then passed to persist callback. - - Requests to Google Analytics can fail, and are retried at most - #{@max_attempts} times with an exponential backoff. Returns `:ok` when - importing has finished or `{:error, term()}` when a request to GA failed too - many times. - - Useful links: - - - [Feature documentation](https://plausible.io/docs/google-analytics-import) - - [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest) - - [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer) - - """ - def import_analytics(date_range, view_id, auth, persist_fn) do - with {:ok, access_token} <- maybe_refresh_token(auth) do - do_import_analytics(date_range, view_id, access_token, persist_fn) - end - end - - defp do_import_analytics(date_range, view_id, access_token, persist_fn) do - Enum.reduce_while(ReportRequest.full_report(), :ok, fn report_request, :ok -> - report_request = %ReportRequest{ - report_request - | date_range: date_range, - view_id: view_id, - access_token: access_token, - page_token: nil, - page_size: @per_page - } - - case fetch_and_persist(report_request, persist_fn: persist_fn) do - :ok -> {:cont, :ok} - {:error, _} = error -> {:halt, error} - end - end) - end - - @spec fetch_and_persist(ReportRequest.t(), Keyword.t()) :: - :ok | {:error, term()} - def fetch_and_persist(%ReportRequest{} = report_request, opts \\ []) do - persist_fn = Keyword.fetch!(opts, :persist_fn) - attempt = Keyword.get(opts, :attempt, 1) - sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor) - - case HTTP.get_report(report_request) do - {:ok, {rows, next_page_token}} -> - :ok = persist_fn.(report_request.dataset, rows) - - if next_page_token do - fetch_and_persist( - %ReportRequest{report_request | page_token: next_page_token}, - opts - ) - else - :ok - end - - {:error, cause} -> - if attempt >= @max_attempts do - {:error, cause} - else - Process.sleep(attempt * sleep_time) - fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1)) - end - end - end - - defp maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do + def maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do with true <- needs_to_refresh_token?(auth.expires), {:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token), changeset <- @@ -200,7 +80,7 @@ defmodule Plausible.Google.Api do end end - defp maybe_refresh_token({access_token, refresh_token, expires_at}) do + def maybe_refresh_token({access_token, refresh_token, expires_at}) do with true <- needs_to_refresh_token?(expires_at), {:ok, {new_access_token, _expires_at}} <- do_refresh_token(refresh_token) do {:ok, new_access_token} diff --git a/lib/plausible/google/ga4/api.ex b/lib/plausible/google/ga4/api.ex new file mode 100644 index 0000000000..f9062d00e3 --- /dev/null +++ b/lib/plausible/google/ga4/api.ex @@ -0,0 +1,142 @@ +defmodule Plausible.Google.GA4.API do + @moduledoc """ + API for Google Analytics 4. + """ + + alias Plausible.Google + alias Plausible.Google.GA4 + + require Logger + + @type import_auth :: { + access_token :: String.t(), + refresh_token :: String.t(), + expires_at :: String.t() + } + + @per_page 50_000 + @backoff_factor :timer.seconds(10) + @max_attempts 5 + + def list_properties(access_token) do + case GA4.HTTP.list_accounts_for_user(access_token) do + {:ok, %{"accountSummaries" => accounts}} -> + accounts = + accounts + |> Enum.filter(& &1["propertySummaries"]) + |> Enum.map(fn account -> + {"#{account["displayName"]} (#{account["account"]})", + Enum.map(account["propertySummaries"], fn property -> + {"#{property["displayName"]} (#{property["property"]})", property["property"]} + end)} + end) + + {:ok, accounts} + + error -> + error + end + end + + def get_property(access_token, lookup_property) do + case list_properties(access_token) do + {:ok, properties} -> + property = + properties + |> Enum.map(&elem(&1, 1)) + |> List.flatten() + |> Enum.find(fn {_name, property} -> property == lookup_property end) + + {:ok, property} + + {:error, cause} -> + {:error, cause} + end + end + + def get_analytics_start_date(access_token, property) do + GA4.HTTP.get_analytics_start_date(access_token, property) + end + + def import_analytics(date_range, property, auth, persist_fn) do + Logger.debug( + "[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}" + ) + + with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do + do_import_analytics(date_range, property, access_token, persist_fn) + end + end + + defp do_import_analytics(date_range, property, access_token, persist_fn) do + Enum.reduce_while(GA4.ReportRequest.full_report(), :ok, fn report_request, :ok -> + Logger.debug( + "[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}" + ) + + report_request = prepare_request(report_request, date_range, property, access_token) + + case fetch_and_persist(report_request, persist_fn: persist_fn) do + :ok -> {:cont, :ok} + {:error, _} = error -> {:halt, error} + end + end) + end + + @spec fetch_and_persist(GA4.ReportRequest.t(), Keyword.t()) :: + :ok | {:error, term()} + def fetch_and_persist(%GA4.ReportRequest{} = report_request, opts \\ []) do + persist_fn = Keyword.fetch!(opts, :persist_fn) + attempt = Keyword.get(opts, :attempt, 1) + sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor) + + case GA4.HTTP.get_report(report_request) do + {:ok, {rows, row_count}} -> + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Fetched #{length(rows)} rows of total #{row_count} with offset #{report_request.offset} for #{report_request.dataset}" + ) + + :ok = persist_fn.(report_request.dataset, rows) + + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Persisted #{length(rows)} for #{report_request.dataset}" + ) + + if report_request.offset + @per_page < row_count do + fetch_and_persist( + %GA4.ReportRequest{report_request | offset: report_request.offset + @per_page}, + opts + ) + else + :ok + end + + {:error, cause} -> + if attempt >= @max_attempts do + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Terminating." + ) + + {:error, cause} + else + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Will retry." + ) + + Process.sleep(attempt * sleep_time) + fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1)) + end + end + end + + defp prepare_request(report_request, date_range, property, access_token) do + %GA4.ReportRequest{ + report_request + | date_range: date_range, + property: property, + access_token: access_token, + offset: 0, + limit: @per_page + } + end +end diff --git a/lib/plausible/google/ga4/http.ex b/lib/plausible/google/ga4/http.ex new file mode 100644 index 0000000000..da9ace19b2 --- /dev/null +++ b/lib/plausible/google/ga4/http.ex @@ -0,0 +1,199 @@ +defmodule Plausible.Google.GA4.HTTP do + @moduledoc """ + HTTP client implementation for Google Analytics 4 API. + """ + + alias Plausible.HTTPClient + + require Logger + + @spec get_report(Plausible.Google.GA4.ReportRequest.t()) :: + {:ok, {[map()], non_neg_integer()}} | {:error, any()} + def get_report(%Plausible.Google.GA4.ReportRequest{} = report_request) do + params = %{ + requests: [ + %{ + property: report_request.property, + dateRanges: [ + %{ + startDate: report_request.date_range.first, + endDate: report_request.date_range.last + } + ], + dimensions: Enum.map(report_request.dimensions, &%{name: &1}), + metrics: Enum.map(report_request.metrics, &build_metric/1), + orderBys: [ + %{ + dimension: %{ + dimensionName: "date", + orderType: "ALPHANUMERIC" + }, + desc: true + } + ], + limit: report_request.limit, + offset: report_request.offset + } + ] + } + + url = + "#{reporting_api_url()}/v1beta/#{report_request.property}:batchRunReports" + + response = + HTTPClient.impl().post( + url, + [{"Authorization", "Bearer #{report_request.access_token}"}], + params, + receive_timeout: 60_000 + ) + + with {:ok, %{body: body}} <- response, + # File.write!("fixture/ga4_report_#{report_request.dataset}.json", Jason.encode!(body)), + {:ok, report} <- parse_report_from_response(body), + row_count <- Map.get(report, "rowCount"), + {:ok, report} <- convert_to_maps(report) do + {:ok, {report, row_count}} + else + {:error, %{reason: %{status: status, body: body}}} -> + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}" + ) + + Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}}) + {:error, :request_failed} + + {:error, reason} -> + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}: #{inspect(reason)}" + ) + + {:error, :request_failed} + end + end + + defp build_metric(expression) do + case String.split(expression, " = ") do + [name, expression] -> + %{ + name: name, + expression: expression + } + + [name] -> + %{name: name} + end + end + + defp parse_report_from_response(%{"reports" => [report | _]}) do + {:ok, report} + end + + defp parse_report_from_response(body) do + Sentry.Context.set_extra_context(%{google_analytics4_response: body}) + + Logger.error( + "Google Analytics 4: Failed to find report in response. Reason: #{inspect(body)}" + ) + + {:error, {:invalid_response, body}} + end + + defp convert_to_maps(%{ + "rows" => rows, + "dimensionHeaders" => dimension_headers, + "metricHeaders" => metric_headers + }) + when is_list(rows) do + dimension_headers = Enum.map(dimension_headers, & &1["name"]) + metric_headers = Enum.map(metric_headers, & &1["name"]) + + report = + Enum.map(rows, fn %{"dimensionValues" => dimensions, "metricValues" => metrics} -> + dimension_values = Enum.map(dimensions, & &1["value"]) + metric_values = Enum.map(metrics, & &1["value"]) + metrics = Enum.zip(metric_headers, metric_values) + dimensions = Enum.zip(dimension_headers, dimension_values) + %{metrics: Map.new(metrics), dimensions: Map.new(dimensions)} + end) + + {:ok, report} + end + + defp convert_to_maps(response) do + Logger.error( + "Google Analytics 4: Failed to read report in response. Reason: #{inspect(response)}" + ) + + Sentry.Context.set_extra_context(%{google_analytics4_response: response}) + {:error, {:invalid_response, response}} + end + + def list_accounts_for_user(access_token) do + url = "#{admin_api_url()}/v1beta/accountSummaries" + + headers = [{"Authorization", "Bearer #{access_token}"}] + + case HTTPClient.impl().get(url, headers) do + {:ok, %Finch.Response{body: body, status: 200}} -> + {:ok, body} + + {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> + {:error, :authentication_failed} + + {:error, %HTTPClient.Non200Error{} = error} -> + Sentry.capture_message("Error listing Google accounts for user", extra: %{error: error}) + {:error, :unknown} + end + end + + @earliest_valid_date "2015-08-14" + def get_analytics_start_date(access_token, property) do + params = %{ + requests: [ + %{ + property: "#{property}", + dateRanges: [ + %{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())} + ], + dimensions: [%{name: "date"}], + metrics: [%{name: "screenPageViews"}], + orderBys: [ + %{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: false} + ], + limit: 1 + } + ] + } + + url = "#{reporting_api_url()}/v1beta/#{property}:batchRunReports" + headers = [{"Authorization", "Bearer #{access_token}"}] + + case HTTPClient.impl().post(url, headers, params) do + {:ok, %Finch.Response{body: body, status: 200}} -> + report = List.first(body["reports"]) + + date = + case report["rows"] do + [%{"dimensionValues" => [%{"value" => date_str}]}] -> + Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date() + + _ -> + nil + end + + {:ok, date} + + {:error, %{reason: %Finch.Response{body: body}}} -> + Sentry.capture_message("Error fetching GA4 start date", extra: %{body: inspect(body)}) + {:error, body} + + {:error, %{reason: reason} = e} -> + Sentry.capture_message("Error fetching GA4 start date", extra: %{error: inspect(e)}) + {:error, reason} + end + end + + defp reporting_api_url, do: "https://analyticsdata.googleapis.com" + defp admin_api_url, do: "https://analyticsadmin.googleapis.com" +end diff --git a/lib/plausible/google/ga4/report_request.ex b/lib/plausible/google/ga4/report_request.ex new file mode 100644 index 0000000000..b859f17893 --- /dev/null +++ b/lib/plausible/google/ga4/report_request.ex @@ -0,0 +1,122 @@ +defmodule Plausible.Google.GA4.ReportRequest do + @moduledoc """ + Report request struct for Google Analytics 4 API + """ + + defstruct [ + :dataset, + :dimensions, + :metrics, + :date_range, + :property, + :access_token, + :offset, + :limit + ] + + @type t() :: %__MODULE__{ + dataset: String.t(), + dimensions: [String.t()], + metrics: [String.t()], + date_range: Date.Range.t(), + property: term(), + access_token: String.t(), + offset: non_neg_integer(), + limit: non_neg_integer() + } + + def full_report do + [ + %__MODULE__{ + dataset: "imported_visitors", + dimensions: ["date"], + metrics: [ + "totalUsers", + "screenPageViews", + "bounces = sessions - engagedSessions", + "sessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_sources", + dimensions: [ + "date", + "sessionSource", + "sessionMedium", + "sessionCampaignName", + "sessionManualAdContent", + "sessionGoogleAdsKeyword" + ], + metrics: [ + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_pages", + dimensions: ["date", "hostName", "pagePath"], + # NOTE: no exits as GA4 DATA API does not provide that metric + metrics: ["totalUsers", "screenPageViews", "userEngagementDuration"] + }, + %__MODULE__{ + dataset: "imported_entry_pages", + dimensions: ["date", "landingPage"], + metrics: [ + "totalUsers", + "sessions", + "userEngagementDuration", + "bounces = sessions - engagedSessions" + ] + }, + # NOTE: Skipping for now as there's no dimension directly mapping to exit page path + # %__MODULE__{ + # dataset: "imported_exit_pages", + # dimensions: ["date", "ga:exitPagePath"], + # metrics: ["totalUsers", "sessions"] + # }, + %__MODULE__{ + dataset: "imported_locations", + dimensions: ["date", "countryId", "region", "city"], + metrics: [ + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_devices", + dimensions: ["date", "deviceCategory"], + metrics: [ + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_browsers", + dimensions: ["date", "browser"], + metrics: [ + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_operating_systems", + dimensions: ["date", "operatingSystem"], + metrics: [ + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + } + ] + end +end diff --git a/lib/plausible/google/http.ex b/lib/plausible/google/http.ex index 93dad5910f..be55fbcf5e 100644 --- a/lib/plausible/google/http.ex +++ b/lib/plausible/google/http.ex @@ -2,97 +2,6 @@ defmodule Plausible.Google.HTTP do require Logger alias Plausible.HTTPClient - @spec get_report(Plausible.Google.ReportRequest.t()) :: - {:ok, {[map()], String.t() | nil}} | {:error, any()} - def get_report(%Plausible.Google.ReportRequest{} = report_request) do - params = %{ - reportRequests: [ - %{ - viewId: report_request.view_id, - dateRanges: [ - %{ - startDate: report_request.date_range.first, - endDate: report_request.date_range.last - } - ], - dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}), - metrics: Enum.map(report_request.metrics, &%{expression: &1}), - hideTotals: true, - hideValueRanges: true, - orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}], - pageSize: report_request.page_size, - pageToken: report_request.page_token - } - ] - } - - response = - HTTPClient.impl().post( - "#{reporting_api_url()}/v4/reports:batchGet", - [{"Authorization", "Bearer #{report_request.access_token}"}], - params, - receive_timeout: 60_000 - ) - - with {:ok, %{body: body}} <- response, - {:ok, report} <- parse_report_from_response(body), - token <- Map.get(report, "nextPageToken"), - {:ok, report} <- convert_to_maps(report) do - {:ok, {report, token}} - else - {:error, %{reason: %{status: status, body: body}}} -> - Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}}) - {:error, :request_failed} - - {:error, _reason} -> - {:error, :request_failed} - end - end - - defp parse_report_from_response(body) do - with %{"reports" => [report | _]} <- body do - {:ok, report} - else - _ -> - Sentry.Context.set_extra_context(%{google_analytics_response: body}) - - Logger.error( - "Google Analytics: Failed to find report in response. Reason: #{inspect(body)}" - ) - - {:error, {:invalid_response, body}} - end - end - - defp convert_to_maps(%{ - "data" => %{} = data, - "columnHeader" => %{ - "dimensions" => dimension_headers, - "metricHeader" => %{"metricHeaderEntries" => metric_headers} - } - }) do - metric_headers = Enum.map(metric_headers, & &1["name"]) - rows = Map.get(data, "rows", []) - - report = - Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} -> - metrics = Enum.zip(metric_headers, metrics) - dimensions = Enum.zip(dimension_headers, dimensions) - %{metrics: Map.new(metrics), dimensions: Map.new(dimensions)} - end) - - {:ok, report} - end - - defp convert_to_maps(response) do - Logger.error( - "Google Analytics: Failed to read report in response. Reason: #{inspect(response)}" - ) - - Sentry.Context.set_extra_context(%{google_analytics_response: response}) - {:error, {:invalid_response, response}} - end - def list_sites(access_token) do url = "#{api_url()}/webmasters/v3/sites" headers = [{"Content-Type", "application/json"}, {"Authorization", "Bearer #{access_token}"}] @@ -113,7 +22,7 @@ defmodule Plausible.Google.HTTP do end end - def fetch_access_token(code) do + def fetch_access_token!(code) do url = "#{api_url()}/oauth2/v4/token" headers = [{"Content-Type", "application/x-www-form-urlencoded"}] @@ -130,24 +39,6 @@ defmodule Plausible.Google.HTTP do response.body end - def list_views_for_user(access_token) do - url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles" - - headers = [{"Authorization", "Bearer #{access_token}"}] - - case HTTPClient.impl().get(url, headers) do - {:ok, %Finch.Response{body: body, status: 200}} -> - {:ok, body} - - {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> - {:error, :authentication_failed} - - {:error, %HTTPClient.Non200Error{} = error} -> - Sentry.capture_message("Error listing GA views for user", extra: %{error: error}) - {:error, :unknown} - end - end - def list_stats(access_token, property, date_range, limit, page \\ nil) do property = URI.encode_www_form(property) @@ -215,57 +106,9 @@ defmodule Plausible.Google.HTTP do end end - @earliest_valid_date "2005-01-01" - def get_analytics_start_date(view_id, access_token) do - params = %{ - reportRequests: [ - %{ - viewId: view_id, - dateRanges: [ - %{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())} - ], - dimensions: [%{name: "ga:date", histogramBuckets: []}], - metrics: [%{expression: "ga:pageviews"}], - hideTotals: true, - hideValueRanges: true, - orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}], - pageSize: 1 - } - ] - } - - url = "#{reporting_api_url()}/v4/reports:batchGet" - headers = [{"Authorization", "Bearer #{access_token}"}] - - case HTTPClient.post(url, headers, params) do - {:ok, %Finch.Response{body: body, status: 200}} -> - report = List.first(body["reports"]) - - date = - case report["data"]["rows"] do - [%{"dimensions" => [date_str]}] -> - Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date() - - _ -> - nil - end - - {:ok, date} - - {:error, %{reason: %Finch.Response{body: body}}} -> - Sentry.capture_message("Error fetching Google view ID", extra: %{body: inspect(body)}) - {:error, body} - - {:error, %{reason: reason} = e} -> - Sentry.capture_message("Error fetching Google view ID", extra: %{error: inspect(e)}) - {:error, reason} - end - end - defp config, do: Application.get_env(:plausible, :google) defp client_id, do: Keyword.fetch!(config(), :client_id) defp client_secret, do: Keyword.fetch!(config(), :client_secret) - defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url) defp api_url, do: Keyword.fetch!(config(), :api_url) defp redirect_uri, do: PlausibleWeb.Endpoint.url() <> "/auth/google/callback" end diff --git a/lib/plausible/google/ua/api.ex b/lib/plausible/google/ua/api.ex new file mode 100644 index 0000000000..47d2cebebf --- /dev/null +++ b/lib/plausible/google/ua/api.ex @@ -0,0 +1,146 @@ +defmodule Plausible.Google.UA.API do + @moduledoc """ + API for Universal Analytics + """ + + alias Plausible.Google + alias Plausible.Google.UA + + @type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()} + + @type import_auth :: { + access_token :: String.t(), + refresh_token :: String.t(), + expires_at :: String.t() + } + + @per_page 7_500 + @backoff_factor :timer.seconds(10) + @max_attempts 5 + + @spec list_views(access_token :: String.t()) :: + {:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()} + @doc """ + Lists Google Analytics views grouped by hostname. + """ + def list_views(access_token) do + case UA.HTTP.list_views_for_user(access_token) do + {:ok, %{"items" => views}} -> + views = Enum.group_by(views, &view_hostname/1, &view_names/1) + {:ok, views} + + error -> + error + end + end + + @spec get_view(access_token :: String.t(), lookup_id :: String.t()) :: + {:ok, google_analytics_view()} | {:ok, nil} | {:error, term()} + @doc """ + Returns a single Google Analytics view if the user has access to it. + """ + def get_view(access_token, lookup_id) do + case list_views(access_token) do + {:ok, views} -> + view = + views + |> Map.values() + |> List.flatten() + |> Enum.find(fn {_name, id} -> id == lookup_id end) + + {:ok, view} + + {:error, cause} -> + {:error, cause} + end + end + + def get_analytics_start_date(access_token, view_id) do + UA.HTTP.get_analytics_start_date(access_token, view_id) + end + + @spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) :: + :ok | {:error, term()} + @doc """ + Imports stats from a Google Analytics UA view to a Plausible site. + + This function fetches Google Analytics reports in batches of #{@per_page} per + request. The batches are then passed to persist callback. + + Requests to Google Analytics can fail, and are retried at most + #{@max_attempts} times with an exponential backoff. Returns `:ok` when + importing has finished or `{:error, term()}` when a request to GA failed too + many times. + + Useful links: + + - [Feature documentation](https://plausible.io/docs/google-analytics-import) + - [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest) + - [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer) + + """ + def import_analytics(date_range, view_id, auth, persist_fn) do + with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do + do_import_analytics(date_range, view_id, access_token, persist_fn) + end + end + + defp do_import_analytics(date_range, view_id, access_token, persist_fn) do + Enum.reduce_while(UA.ReportRequest.full_report(), :ok, fn report_request, :ok -> + report_request = %UA.ReportRequest{ + report_request + | date_range: date_range, + view_id: view_id, + access_token: access_token, + page_token: nil, + page_size: @per_page + } + + case fetch_and_persist(report_request, persist_fn: persist_fn) do + :ok -> {:cont, :ok} + {:error, _} = error -> {:halt, error} + end + end) + end + + @spec fetch_and_persist(UA.ReportRequest.t(), Keyword.t()) :: + :ok | {:error, term()} + def fetch_and_persist(%UA.ReportRequest{} = report_request, opts \\ []) do + persist_fn = Keyword.fetch!(opts, :persist_fn) + attempt = Keyword.get(opts, :attempt, 1) + sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor) + + case UA.HTTP.get_report(report_request) do + {:ok, {rows, next_page_token}} -> + :ok = persist_fn.(report_request.dataset, rows) + + if next_page_token do + fetch_and_persist( + %UA.ReportRequest{report_request | page_token: next_page_token}, + opts + ) + else + :ok + end + + {:error, cause} -> + if attempt >= @max_attempts do + {:error, cause} + else + Process.sleep(attempt * sleep_time) + fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1)) + end + end + end + + defp view_hostname(view) do + case view do + %{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host) + _any -> "Others" + end + end + + defp view_names(%{"name" => name, "id" => id}) do + {"#{id} - #{name}", id} + end +end diff --git a/lib/plausible/google/ua/http.ex b/lib/plausible/google/ua/http.ex new file mode 100644 index 0000000000..f00712f5ec --- /dev/null +++ b/lib/plausible/google/ua/http.ex @@ -0,0 +1,167 @@ +defmodule Plausible.Google.UA.HTTP do + @moduledoc """ + HTTP client implementation for Universal Analytics API. + """ + + require Logger + alias Plausible.HTTPClient + + @spec get_report(Plausible.Google.UA.ReportRequest.t()) :: + {:ok, {[map()], String.t() | nil}} | {:error, any()} + def get_report(%Plausible.Google.UA.ReportRequest{} = report_request) do + params = %{ + reportRequests: [ + %{ + viewId: report_request.view_id, + dateRanges: [ + %{ + startDate: report_request.date_range.first, + endDate: report_request.date_range.last + } + ], + dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}), + metrics: Enum.map(report_request.metrics, &%{expression: &1}), + hideTotals: true, + hideValueRanges: true, + orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}], + pageSize: report_request.page_size, + pageToken: report_request.page_token + } + ] + } + + response = + HTTPClient.impl().post( + "#{reporting_api_url()}/v4/reports:batchGet", + [{"Authorization", "Bearer #{report_request.access_token}"}], + params, + receive_timeout: 60_000 + ) + + with {:ok, %{body: body}} <- response, + {:ok, report} <- parse_report_from_response(body), + token <- Map.get(report, "nextPageToken"), + {:ok, report} <- convert_to_maps(report) do + {:ok, {report, token}} + else + {:error, %{reason: %{status: status, body: body}}} -> + Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}}) + {:error, :request_failed} + + {:error, _reason} -> + {:error, :request_failed} + end + end + + defp parse_report_from_response(%{"reports" => [report | _]}) do + {:ok, report} + end + + defp parse_report_from_response(body) do + Sentry.Context.set_extra_context(%{universal_analytics_response: body}) + + Logger.error( + "Universal Analytics: Failed to find report in response. Reason: #{inspect(body)}" + ) + + {:error, {:invalid_response, body}} + end + + defp convert_to_maps(%{ + "data" => %{} = data, + "columnHeader" => %{ + "dimensions" => dimension_headers, + "metricHeader" => %{"metricHeaderEntries" => metric_headers} + } + }) do + metric_headers = Enum.map(metric_headers, & &1["name"]) + rows = Map.get(data, "rows", []) + + report = + Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} -> + metrics = Enum.zip(metric_headers, metrics) + dimensions = Enum.zip(dimension_headers, dimensions) + %{metrics: Map.new(metrics), dimensions: Map.new(dimensions)} + end) + + {:ok, report} + end + + defp convert_to_maps(response) do + Logger.error( + "Universal Analytics: Failed to read report in response. Reason: #{inspect(response)}" + ) + + Sentry.Context.set_extra_context(%{universal_analytics_response: response}) + {:error, {:invalid_response, response}} + end + + def list_views_for_user(access_token) do + url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles" + + headers = [{"Authorization", "Bearer #{access_token}"}] + + case HTTPClient.impl().get(url, headers) do + {:ok, %Finch.Response{body: body, status: 200}} -> + {:ok, body} + + {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> + {:error, :authentication_failed} + + {:error, %HTTPClient.Non200Error{} = error} -> + Sentry.capture_message("Error listing GA views for user", extra: %{error: error}) + {:error, :unknown} + end + end + + @earliest_valid_date "2005-01-01" + def get_analytics_start_date(access_token, view_id) do + params = %{ + reportRequests: [ + %{ + viewId: view_id, + dateRanges: [ + %{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())} + ], + dimensions: [%{name: "ga:date", histogramBuckets: []}], + metrics: [%{expression: "ga:pageviews"}], + hideTotals: true, + hideValueRanges: true, + orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}], + pageSize: 1 + } + ] + } + + url = "#{reporting_api_url()}/v4/reports:batchGet" + headers = [{"Authorization", "Bearer #{access_token}"}] + + case HTTPClient.post(url, headers, params) do + {:ok, %Finch.Response{body: body, status: 200}} -> + report = List.first(body["reports"]) + + date = + case report["data"]["rows"] do + [%{"dimensions" => [date_str]}] -> + Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date() + + _ -> + nil + end + + {:ok, date} + + {:error, %{reason: %Finch.Response{body: body}}} -> + Sentry.capture_message("Error fetching UA start date", extra: %{body: inspect(body)}) + {:error, body} + + {:error, %{reason: reason} = e} -> + Sentry.capture_message("Error fetching UA start date", extra: %{error: inspect(e)}) + {:error, reason} + end + end + + defp config, do: Application.get_env(:plausible, :google) + defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url) + defp api_url, do: Keyword.fetch!(config(), :api_url) +end diff --git a/lib/plausible/google/report_request.ex b/lib/plausible/google/ua/report_request.ex similarity index 95% rename from lib/plausible/google/report_request.ex rename to lib/plausible/google/ua/report_request.ex index fcc8fe2c12..e55419dd7d 100644 --- a/lib/plausible/google/report_request.ex +++ b/lib/plausible/google/ua/report_request.ex @@ -1,4 +1,8 @@ -defmodule Plausible.Google.ReportRequest do +defmodule Plausible.Google.UA.ReportRequest do + @moduledoc """ + Report request struct for Universal Analytics API + """ + defstruct [ :dataset, :dimensions, diff --git a/lib/plausible/imported/buffer.ex b/lib/plausible/imported/buffer.ex index 554cfaf701..fccbb62784 100644 --- a/lib/plausible/imported/buffer.ex +++ b/lib/plausible/imported/buffer.ex @@ -8,12 +8,13 @@ defmodule Plausible.Imported.Buffer do use GenServer require Logger - def start_link do - GenServer.start_link(__MODULE__, nil) + def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, opts) end - def init(_opts) do - {:ok, %{buffers: %{}}} + def init(opts) do + flush_interval = Keyword.get(opts, :flush_interval_ms, 1000) + {:ok, %{flush_interval: flush_interval, buffers: %{}}} end @spec insert_many(pid(), term(), [map()]) :: :ok @@ -68,14 +69,14 @@ defmodule Plausible.Imported.Buffer do def handle_call(:flush_all_buffers, _from, state) do Enum.each(state.buffers, fn {table_name, records} -> - flush_buffer(records, table_name) + flush_buffer(records, table_name, state.flush_interval) end) {:reply, :ok, put_in(state.buffers, %{})} end def handle_continue({:flush, table_name}, state) do - flush_buffer(state.buffers[table_name], table_name) + flush_buffer(state.buffers[table_name], table_name, state.flush_interval) {:noreply, put_in(state.buffers[table_name], [])} end @@ -85,10 +86,10 @@ defmodule Plausible.Imported.Buffer do |> Keyword.fetch!(:max_buffer_size) end - defp flush_buffer(records, table_name) do + defp flush_buffer(records, table_name, flush_interval) do # Clickhouse does not recommend sending more than 1 INSERT operation per second, and this # sleep call slows down the flushing - Process.sleep(1000) + Process.sleep(flush_interval) Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer") insert_all(table_name, records) diff --git a/lib/plausible/imported/google_analytics4.ex b/lib/plausible/imported/google_analytics4.ex new file mode 100644 index 0000000000..5832421334 --- /dev/null +++ b/lib/plausible/imported/google_analytics4.ex @@ -0,0 +1,253 @@ +defmodule Plausible.Imported.GoogleAnalytics4 do + @moduledoc """ + Import implementation for Google Analytics 4. + """ + + use Plausible.Imported.Importer + + @missing_values ["(none)", "(not set)", "(not provided)", "(other)"] + + @impl true + def name(), do: :google_analytics_4 + + @impl true + def label(), do: "Google Analytics 4" + + @impl true + def email_template(), do: "google_analytics_import.html" + + @impl true + def parse_args( + %{"property" => property, "start_date" => start_date, "end_date" => end_date} = args + ) do + start_date = Date.from_iso8601!(start_date) + end_date = Date.from_iso8601!(end_date) + date_range = Date.range(start_date, end_date) + + auth = { + Map.fetch!(args, "access_token"), + Map.fetch!(args, "refresh_token"), + Map.fetch!(args, "token_expires_at") + } + + [ + property: property, + date_range: date_range, + auth: auth + ] + end + + @doc """ + Imports stats from a Google Analytics 4 property to a Plausible site. + + This function fetches Google Analytics 4 reports which are then passed in batches + to Clickhouse by the `Plausible.Imported.Buffer` process. + """ + @impl true + def import_data(site_import, opts) do + date_range = Keyword.fetch!(opts, :date_range) + property = Keyword.fetch!(opts, :property) + auth = Keyword.fetch!(opts, :auth) + flush_interval_ms = Keyword.get(opts, :flush_interval_ms, 1000) + + {:ok, buffer} = Plausible.Imported.Buffer.start_link(flush_interval_ms: flush_interval_ms) + + persist_fn = fn table, rows -> + records = from_report(rows, site_import.site_id, site_import.id, table) + Plausible.Imported.Buffer.insert_many(buffer, table, records) + end + + try do + Plausible.Google.GA4.API.import_analytics(date_range, property, auth, persist_fn) + after + Plausible.Imported.Buffer.flush(buffer) + Plausible.Imported.Buffer.stop(buffer) + end + end + + def from_report(nil, _site_id, _import_id, _metric), do: nil + + def from_report(data, site_id, import_id, table) do + Enum.reduce(data, [], fn row, acc -> + if Map.get(row.dimensions, "date") in @missing_values do + acc + else + [new_from_report(site_id, import_id, table, row) | acc] + end + end) + end + + defp parse_number(nr) do + {float, ""} = Float.parse(nr) + round(float) + end + + defp new_from_report(site_id, import_id, "imported_visitors", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + defp new_from_report(site_id, import_id, "imported_sources", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(), + utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(), + utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(), + utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(), + utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + defp new_from_report(site_id, import_id, "imported_pages", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""), + page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), + # NOTE: no exits metric in GA4 API currently + exits: 0, + time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + defp new_from_report(site_id, import_id, "imported_entry_pages", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + entry_page: row.dimensions |> Map.fetch!("landingPage"), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number() + } + end + + # NOTE: no exit pages metrics in GA4 API available for now + # defp new_from_report(site_id, import_id, "imported_exit_pages", row) do + # %{ + # site_id: site_id, + # import_id: import_id, + # date: get_date(row), + # exit_page: Map.fetch!(row.dimensions, "exitPage"), + # visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + # exits: row.metrics |> Map.fetch!("sessions") |> parse_number() + # } + # end + + defp new_from_report(site_id, import_id, "imported_locations", row) do + country_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("") + city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("") + city_data = Location.get_city(city_name, country_code) + + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + country: country_code, + region: row.dimensions |> Map.fetch!("region") |> default_if_missing(""), + city: city_data && city_data.id, + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + defp new_from_report(site_id, import_id, "imported_devices", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + @browser_google_to_plausible %{ + "User-Agent:Opera" => "Opera", + "Mozilla Compatible Agent" => "Mobile App", + "Android Webview" => "Mobile App", + "Android Browser" => "Mobile App", + "Safari (in-app)" => "Mobile App", + "User-Agent: Mozilla" => "Firefox", + "(not set)" => "" + } + + defp new_from_report(site_id, import_id, "imported_browsers", row) do + browser = Map.fetch!(row.dimensions, "browser") + + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + browser: Map.get(@browser_google_to_plausible, browser, browser), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + @os_google_to_plausible %{ + "Macintosh" => "Mac", + "Linux" => "GNU/Linux", + "(not set)" => "" + } + + defp new_from_report(site_id, import_id, "imported_operating_systems", row) do + os = Map.fetch!(row.dimensions, "operatingSystem") + + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + operating_system: Map.get(@os_google_to_plausible, os, os), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + defp get_date(%{dimensions: %{"date" => date}}) do + date + |> Timex.parse!("%Y%m%d", :strftime) + |> NaiveDateTime.to_date() + end + + defp default_if_missing(value, default \\ nil) + defp default_if_missing(value, default) when value in @missing_values, do: default + defp default_if_missing(value, _default), do: value + + defp parse_referrer(nil), do: nil + defp parse_referrer("(direct)"), do: nil + defp parse_referrer("google"), do: "Google" + defp parse_referrer("bing"), do: "Bing" + defp parse_referrer("duckduckgo"), do: "DuckDuckGo" + + defp parse_referrer(ref) do + RefInspector.parse("https://" <> ref) + |> PlausibleWeb.RefInspector.parse() + end +end diff --git a/lib/plausible/imported/import_sources.ex b/lib/plausible/imported/import_sources.ex index bcf1eb8d45..96a149b19b 100644 --- a/lib/plausible/imported/import_sources.ex +++ b/lib/plausible/imported/import_sources.ex @@ -4,6 +4,7 @@ defmodule Plausible.Imported.ImportSources do """ @sources [ + Plausible.Imported.GoogleAnalytics4, Plausible.Imported.UniversalAnalytics, Plausible.Imported.NoopImporter, Plausible.Imported.CSVImporter diff --git a/lib/plausible/imported/universal_analytics.ex b/lib/plausible/imported/universal_analytics.ex index 1c7a1e093e..1d0dcb2e8a 100644 --- a/lib/plausible/imported/universal_analytics.ex +++ b/lib/plausible/imported/universal_analytics.ex @@ -102,7 +102,7 @@ defmodule Plausible.Imported.UniversalAnalytics do end try do - Plausible.Google.Api.import_analytics(date_range, view_id, auth, persist_fn) + Plausible.Google.UA.API.import_analytics(date_range, view_id, auth, persist_fn) after Plausible.Imported.Buffer.flush(buffer) Plausible.Imported.Buffer.stop(buffer) diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.ex index 11a68efc52..a242413710 100644 --- a/lib/plausible_web/controllers/auth_controller.ex +++ b/lib/plausible_web/controllers/auth_controller.ex @@ -697,13 +697,16 @@ defmodule PlausibleWeb.AuthController do end def google_auth_callback(conn, %{"error" => error, "state" => state} = params) do - [site_id, _redirected_to, legacy] = + [site_id, _redirected_to, legacy, _ga4] = case Jason.decode!(state) do [site_id, redirect_to] -> - [site_id, redirect_to, true] + [site_id, redirect_to, true, false] [site_id, redirect_to, legacy] -> - [site_id, redirect_to, legacy] + [site_id, redirect_to, legacy, false] + + [site_id, redirect_to, legacy, ga4] -> + [site_id, redirect_to, legacy, ga4] end site = Repo.get(Plausible.Site, site_id) @@ -745,15 +748,18 @@ defmodule PlausibleWeb.AuthController do end def google_auth_callback(conn, %{"code" => code, "state" => state}) do - res = Plausible.Google.HTTP.fetch_access_token(code) + res = Plausible.Google.API.fetch_access_token!(code) - [site_id, redirect_to, legacy] = + [site_id, redirect_to, legacy, ga4] = case Jason.decode!(state) do [site_id, redirect_to] -> - [site_id, redirect_to, true] + [site_id, redirect_to, true, false] [site_id, redirect_to, legacy] -> - [site_id, redirect_to, legacy] + [site_id, redirect_to, legacy, false] + + [site_id, redirect_to, legacy, ga4] -> + [site_id, redirect_to, legacy, ga4] end site = Repo.get(Plausible.Site, site_id) @@ -761,15 +767,26 @@ defmodule PlausibleWeb.AuthController do case redirect_to do "import" -> - redirect(conn, - external: - Routes.site_path(conn, :import_from_google_view_id_form, site.domain, - access_token: res["access_token"], - refresh_token: res["refresh_token"], - expires_at: NaiveDateTime.to_iso8601(expires_at), - legacy: legacy - ) - ) + if ga4 do + redirect(conn, + external: + Routes.google_analytics4_path(conn, :property_form, site.domain, + access_token: res["access_token"], + refresh_token: res["refresh_token"], + expires_at: NaiveDateTime.to_iso8601(expires_at) + ) + ) + else + redirect(conn, + external: + Routes.universal_analytics_path(conn, :view_id_form, site.domain, + access_token: res["access_token"], + refresh_token: res["refresh_token"], + expires_at: NaiveDateTime.to_iso8601(expires_at), + legacy: legacy + ) + ) + end _ -> id_token = res["id_token"] diff --git a/lib/plausible_web/controllers/google_analytics4_controller.ex b/lib/plausible_web/controllers/google_analytics4_controller.ex new file mode 100644 index 0000000000..3fd55d3ab6 --- /dev/null +++ b/lib/plausible_web/controllers/google_analytics4_controller.ex @@ -0,0 +1,143 @@ +defmodule PlausibleWeb.GoogleAnalytics4Controller do + use PlausibleWeb, :controller + + plug(PlausibleWeb.RequireAccountPlug) + + plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin]) + + def property_form(conn, %{ + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at + }) do + redirect_route = Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain) + + case Plausible.Google.GA4.API.list_properties(access_token) do + {:ok, properties} -> + conn + |> assign(:skip_plausible_tracking, true) + |> render("property_form.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: conn.assigns.site, + properties: properties, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + + {:error, :authentication_failed} -> + conn + |> put_flash( + :error, + "We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again." + ) + |> redirect(external: redirect_route) + + {:error, _any} -> + conn + |> put_flash( + :error, + "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance." + ) + |> redirect(external: redirect_route) + end + end + + def property(conn, %{ + "property" => property, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at + }) do + site = conn.assigns.site + start_date = Plausible.Google.GA4.API.get_analytics_start_date(access_token, property) + + case start_date do + {:ok, nil} -> + {:ok, properties} = Plausible.Google.GA4.API.list_properties(access_token) + + conn + |> assign(:skip_plausible_tracking, true) + |> render("property_form.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: site, + properties: properties, + selected_property_error: "No data found. Nothing to import", + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + + {:ok, _date} -> + redirect(conn, + to: + Routes.google_analytics4_path(conn, :confirm, site.domain, + property: property, + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at + ) + ) + end + end + + def confirm(conn, %{ + "property" => property, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at + }) do + site = conn.assigns.site + + start_date = Plausible.Google.GA4.API.get_analytics_start_date(access_token, property) + + end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone) + + {:ok, {property_name, property}} = + Plausible.Google.GA4.API.get_property(access_token, property) + + conn + |> assign(:skip_plausible_tracking, true) + |> render("confirm.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: site, + selected_property: property, + selected_property_name: property_name, + start_date: start_date, + end_date: end_date, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + end + + def import(conn, %{ + "property" => property, + "start_date" => start_date, + "end_date" => end_date, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at + }) do + site = conn.assigns.site + current_user = conn.assigns.current_user + + redirect_route = Routes.site_path(conn, :settings_imports_exports, site.domain) + + {:ok, _} = + Plausible.Imported.GoogleAnalytics4.new_import( + site, + current_user, + property: property, + start_date: start_date, + end_date: end_date, + access_token: access_token, + refresh_token: refresh_token, + token_expires_at: expires_at + ) + + conn + |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") + |> redirect(external: redirect_route) + end +end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 51c04c7991..aa38531cc7 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -232,7 +232,7 @@ defmodule PlausibleWeb.SiteController do search_console_domains = if site.google_auth do - Plausible.Google.Api.fetch_verified_properties(site.google_auth) + Plausible.Google.API.fetch_verified_properties(site.google_auth) end imported_pageviews = @@ -641,198 +641,6 @@ defmodule PlausibleWeb.SiteController do end end - def import_from_google_user_metric_notice(conn, %{ - "view_id" => view_id, - "access_token" => access_token, - "refresh_token" => refresh_token, - "expires_at" => expires_at, - "legacy" => legacy - }) do - site = conn.assigns[:site] - - conn - |> assign(:skip_plausible_tracking, true) - |> render("import_from_google_user_metric_form.html", - site: site, - view_id: view_id, - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - legacy: legacy, - layout: {PlausibleWeb.LayoutView, "focus.html"} - ) - end - - def import_from_google_view_id_form(conn, %{ - "access_token" => access_token, - "refresh_token" => refresh_token, - "expires_at" => expires_at, - "legacy" => legacy - }) do - redirect_route = - if legacy == "true" do - Routes.site_path(conn, :settings_integrations, conn.assigns.site.domain) - else - Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain) - end - - case Plausible.Google.Api.list_views(access_token) do - {:ok, view_ids} -> - conn - |> assign(:skip_plausible_tracking, true) - |> render("import_from_google_view_id_form.html", - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - site: conn.assigns.site, - view_ids: view_ids, - legacy: legacy, - layout: {PlausibleWeb.LayoutView, "focus.html"} - ) - - {:error, :authentication_failed} -> - conn - |> put_flash( - :error, - "We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again." - ) - |> redirect(external: redirect_route) - - {:error, _any} -> - conn - |> put_flash( - :error, - "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance." - ) - |> redirect(external: redirect_route) - end - end - - # see https://stackoverflow.com/a/57416769 - @google_analytics_new_user_metric_date ~D[2016-08-24] - def import_from_google_view_id(conn, %{ - "view_id" => view_id, - "access_token" => access_token, - "refresh_token" => refresh_token, - "expires_at" => expires_at, - "legacy" => legacy - }) do - site = conn.assigns[:site] - start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token) - - case start_date do - {:ok, nil} -> - site = conn.assigns[:site] - {:ok, view_ids} = Plausible.Google.Api.list_views(access_token) - - conn - |> assign(:skip_plausible_tracking, true) - |> render("import_from_google_view_id_form.html", - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - site: site, - view_ids: view_ids, - selected_view_id_error: "No data found. Nothing to import", - legacy: legacy, - layout: {PlausibleWeb.LayoutView, "focus.html"} - ) - - {:ok, date} -> - if Timex.before?(date, @google_analytics_new_user_metric_date) do - redirect(conn, - to: - Routes.site_path(conn, :import_from_google_user_metric_notice, site.domain, - view_id: view_id, - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - legacy: legacy - ) - ) - else - redirect(conn, - to: - Routes.site_path(conn, :import_from_google_confirm, site.domain, - view_id: view_id, - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - legacy: legacy - ) - ) - end - end - end - - def import_from_google_confirm(conn, %{ - "view_id" => view_id, - "access_token" => access_token, - "refresh_token" => refresh_token, - "expires_at" => expires_at, - "legacy" => legacy - }) do - site = conn.assigns[:site] - - start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token) - - end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone) - - {:ok, {view_name, view_id}} = Plausible.Google.Api.get_view(access_token, view_id) - - conn - |> assign(:skip_plausible_tracking, true) - |> render("import_from_google_confirm.html", - access_token: access_token, - refresh_token: refresh_token, - expires_at: expires_at, - site: site, - selected_view_id: view_id, - selected_view_id_name: view_name, - start_date: start_date, - end_date: end_date, - legacy: legacy, - layout: {PlausibleWeb.LayoutView, "focus.html"} - ) - end - - def import_from_google(conn, %{ - "view_id" => view_id, - "start_date" => start_date, - "end_date" => end_date, - "access_token" => access_token, - "refresh_token" => refresh_token, - "expires_at" => expires_at, - "legacy" => legacy - }) do - site = conn.assigns.site - current_user = conn.assigns.current_user - - redirect_route = - if legacy == "true" do - Routes.site_path(conn, :settings_integrations, site.domain) - else - Routes.site_path(conn, :settings_imports_exports, site.domain) - end - - {:ok, _} = - Plausible.Imported.UniversalAnalytics.new_import( - site, - current_user, - view_id: view_id, - start_date: start_date, - end_date: end_date, - access_token: access_token, - refresh_token: refresh_token, - token_expires_at: expires_at, - legacy: legacy == "true" - ) - - conn - |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") - |> redirect(external: redirect_route) - end - def forget_import(conn, %{"import_id" => import_id}) do site = conn.assigns.site diff --git a/lib/plausible_web/controllers/universal_analytics_controller.ex b/lib/plausible_web/controllers/universal_analytics_controller.ex new file mode 100644 index 0000000000..16a4bea64e --- /dev/null +++ b/lib/plausible_web/controllers/universal_analytics_controller.ex @@ -0,0 +1,198 @@ +defmodule PlausibleWeb.UniversalAnalyticsController do + use PlausibleWeb, :controller + + plug(PlausibleWeb.RequireAccountPlug) + + plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin]) + + def user_metric_notice(conn, %{ + "view_id" => view_id, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "legacy" => legacy + }) do + site = conn.assigns.site + + conn + |> assign(:skip_plausible_tracking, true) + |> render("user_metric_form.html", + site: site, + view_id: view_id, + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + legacy: legacy, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + end + + def view_id_form(conn, %{ + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "legacy" => legacy + }) do + redirect_route = + if legacy == "true" do + Routes.site_path(conn, :settings_integrations, conn.assigns.site.domain) + else + Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain) + end + + case Plausible.Google.UA.API.list_views(access_token) do + {:ok, view_ids} -> + conn + |> assign(:skip_plausible_tracking, true) + |> render("view_id_form.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: conn.assigns.site, + view_ids: view_ids, + legacy: legacy, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + + {:error, :authentication_failed} -> + conn + |> put_flash( + :error, + "We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again." + ) + |> redirect(external: redirect_route) + + {:error, _any} -> + conn + |> put_flash( + :error, + "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance." + ) + |> redirect(external: redirect_route) + end + end + + # see https://stackoverflow.com/a/57416769 + @google_analytics_new_user_metric_date ~D[2016-08-24] + def view_id(conn, %{ + "view_id" => view_id, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "legacy" => legacy + }) do + site = conn.assigns.site + start_date = Plausible.Google.UA.API.get_analytics_start_date(access_token, view_id) + + case start_date do + {:ok, nil} -> + {:ok, view_ids} = Plausible.Google.UA.API.list_views(access_token) + + conn + |> assign(:skip_plausible_tracking, true) + |> render("view_id_form.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: site, + view_ids: view_ids, + selected_view_id_error: "No data found. Nothing to import", + legacy: legacy, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + + {:ok, date} -> + if Timex.before?(date, @google_analytics_new_user_metric_date) do + redirect(conn, + to: + Routes.universal_analytics_path(conn, :user_metric_notice, site.domain, + view_id: view_id, + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + legacy: legacy + ) + ) + else + redirect(conn, + to: + Routes.universal_analytics_path(conn, :confirm, site.domain, + view_id: view_id, + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + legacy: legacy + ) + ) + end + end + end + + def confirm(conn, %{ + "view_id" => view_id, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "legacy" => legacy + }) do + site = conn.assigns.site + + start_date = Plausible.Google.UA.API.get_analytics_start_date(access_token, view_id) + + end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone) + + {:ok, {view_name, view_id}} = Plausible.Google.UA.API.get_view(access_token, view_id) + + conn + |> assign(:skip_plausible_tracking, true) + |> render("confirm.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: site, + selected_view_id: view_id, + selected_view_id_name: view_name, + start_date: start_date, + end_date: end_date, + legacy: legacy, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + end + + def import(conn, %{ + "view_id" => view_id, + "start_date" => start_date, + "end_date" => end_date, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "legacy" => legacy + }) do + site = conn.assigns.site + current_user = conn.assigns.current_user + + redirect_route = + if legacy == "true" do + Routes.site_path(conn, :settings_integrations, site.domain) + else + Routes.site_path(conn, :settings_imports_exports, site.domain) + end + + {:ok, _} = + Plausible.Imported.UniversalAnalytics.new_import( + site, + current_user, + view_id: view_id, + start_date: start_date, + end_date: end_date, + access_token: access_token, + refresh_token: refresh_token, + token_expires_at: expires_at, + legacy: legacy == "true" + ) + + conn + |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") + |> redirect(external: redirect_route) + end +end diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index d1c414ac76..162dee3cad 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -374,17 +374,27 @@ defmodule PlausibleWeb.Router do delete "/:website/stats", SiteController, :reset_stats get "/:website/import/google-analytics/view-id", - SiteController, - :import_from_google_view_id_form + UniversalAnalyticsController, + :view_id_form - post "/:website/import/google-analytics/view-id", SiteController, :import_from_google_view_id + post "/:website/import/google-analytics/view-id", UniversalAnalyticsController, :view_id get "/:website/import/google-analytics/user-metric", - SiteController, - :import_from_google_user_metric_notice + UniversalAnalyticsController, + :user_metric_notice + + get "/:website/import/google-analytics/confirm", UniversalAnalyticsController, :confirm + post "/:website/settings/google-import", UniversalAnalyticsController, :import + + get "/:website/import/google-analytics4/property", + GoogleAnalytics4Controller, + :property_form + + post "/:website/import/google-analytics4/property", GoogleAnalytics4Controller, :property + + get "/:website/import/google-analytics4/confirm", GoogleAnalytics4Controller, :confirm + post "/:website/settings/google4-import", GoogleAnalytics4Controller, :import - get "/:website/import/google-analytics/confirm", SiteController, :import_from_google_confirm - post "/:website/settings/google-import", SiteController, :import_from_google delete "/:website/settings/forget-imported", SiteController, :forget_imported delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import diff --git a/lib/plausible_web/templates/google_analytics4/confirm.html.heex b/lib/plausible_web/templates/google_analytics4/confirm.html.heex new file mode 100644 index 0000000000..b0f63048ba --- /dev/null +++ b/lib/plausible_web/templates/google_analytics4/confirm.html.heex @@ -0,0 +1,46 @@ +<%= form_for @conn, Routes.google_analytics4_path(@conn, :import, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %> +

Import from Google Analytics

+ + <%= hidden_input(f, :access_token, value: @access_token) %> + <%= hidden_input(f, :refresh_token, value: @refresh_token) %> + <%= hidden_input(f, :expires_at, value: @expires_at) %> + + <%= case @start_date do %> + <% {:ok, start_date} -> %> +
+ Stats from this property and time period will be imported from your Google Analytics 4 account to your Plausible dashboard +
+ +
+ <%= styled_label(f, :property, "Google Analytics 4 property") %> + + <%= @selected_property_name %> + + <%= hidden_input(f, :property, readonly: "true", value: @selected_property) %> +
+
+
+ <%= styled_label(f, :start_date, "From") %> + + <%= PlausibleWeb.EmailView.date_format(start_date) %> + + <%= hidden_input(f, :start_date, value: start_date, readonly: "true") %> +
+
→
+
+ <%= styled_label(f, :end_date, "To") %> + + <%= PlausibleWeb.EmailView.date_format(@end_date) %> + + <%= hidden_input(f, :end_date, value: @end_date, readonly: "true") %> +
+
+ <% {:error, error} -> %> +

+ The following error occurred when fetching your Google Analytics 4 data. +

+

<%= error %>

+ <% end %> + + <%= submit("Confirm import", class: "button mt-6") %> +<% end %> diff --git a/lib/plausible_web/templates/google_analytics4/property_form.html.heex b/lib/plausible_web/templates/google_analytics4/property_form.html.heex new file mode 100644 index 0000000000..c88461073c --- /dev/null +++ b/lib/plausible_web/templates/google_analytics4/property_form.html.heex @@ -0,0 +1,19 @@ +<%= form_for @conn, Routes.google_analytics4_path(@conn, :property, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %> +

Import from Google Analytics 4

+ + <%= hidden_input(f, :access_token, value: @access_token) %> + <%= hidden_input(f, :refresh_token, value: @refresh_token) %> + <%= hidden_input(f, :expires_at, value: @expires_at) %> + +
+ Choose the property in your Google Analytics 4 account that will be imported to the <%= @site.domain %> dashboard. +
+ +
+ <%= styled_label(f, :property, "Google Analytics 4 property") %> + <%= styled_select(f, :property, @properties, prompt: "(Choose property)", required: "true") %> + <%= styled_error(@conn.assigns[:selected_property_error]) %> +
+ + <%= submit("Continue ->", class: "button mt-6") %> +<% end %> diff --git a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex b/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex deleted file mode 100644 index e5792ed9f9..0000000000 --- a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex +++ /dev/null @@ -1,26 +0,0 @@ -
-

Import from Google Analytics

- -
-

- - - - - Important: Since your GA property includes data from before 23rd August 2016, you have to take an extra step to make sure we can import data smoothly. -

- -
    -
  1. 1. Navigate to the GA property you want to import from
    2. -
  2. 2. Go to Admin > Property Settings > User Analysis
    3. -
  3. 3. Make sure Enable Users Metric in Reporting is OFF
    4. -
- -

- The setting may take a few minutes to take effect. If your imported data is showing 0 visitors in unexpected places, it's probably caused by this and you - can try importing again later. -

-
- - <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token, refresh_token: @refresh_token, expires_at: @expires_at, legacy: @legacy), class: "button mt-6") %> -
diff --git a/lib/plausible_web/templates/site/settings_google_import.html.heex b/lib/plausible_web/templates/site/settings_google_import.html.heex index fb223f2982..c7d76b99c8 100644 --- a/lib/plausible_web/templates/site/settings_google_import.html.heex +++ b/lib/plausible_web/templates/site/settings_google_import.html.heex @@ -92,7 +92,7 @@ <% end %> <% end %> <% else %> diff --git a/lib/plausible_web/templates/site/settings_imports_exports.html.heex b/lib/plausible_web/templates/site/settings_imports_exports.html.heex index 9369143a17..b1b31f877c 100644 --- a/lib/plausible_web/templates/site/settings_imports_exports.html.heex +++ b/lib/plausible_web/templates/site/settings_imports_exports.html.heex @@ -14,20 +14,22 @@ New Universal Analytics import New Universal Analytics import diff --git a/lib/plausible_web/templates/site/settings_search_console.html.heex b/lib/plausible_web/templates/site/settings_search_console.html.heex index 5b5b6aff9c..c615fab3aa 100644 --- a/lib/plausible_web/templates/site/settings_search_console.html.heex +++ b/lib/plausible_web/templates/site/settings_search_console.html.heex @@ -82,7 +82,7 @@ <% else %>
NB: You also need to set up your site on diff --git a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex b/lib/plausible_web/templates/universal_analytics/confirm.html.heex similarity index 52% rename from lib/plausible_web/templates/site/import_from_google_confirm.html.eex rename to lib/plausible_web/templates/universal_analytics/confirm.html.heex index c46459aa94..3d6f143f32 100644 --- a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex +++ b/lib/plausible_web/templates/universal_analytics/confirm.html.heex @@ -1,4 +1,4 @@ -<%= form_for @conn, Routes.site_path(@conn, :import_from_google, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %> +<%= form_for @conn, Routes.universal_analytics_path(@conn, :import, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>

Import from Google Analytics

<%= hidden_input(f, :access_token, value: @access_token) %> @@ -8,33 +8,40 @@ <%= case @start_date do %> <% {:ok, start_date} -> %> -
Stats from this property and time period will be imported from your Google Analytics account to your Plausible dashboard
<%= styled_label(f, :view_id, "Google Analytics view") %> - <%= @selected_view_id_name %> - <%= hidden_input f, :view_id, readonly: "true", value: @selected_view_id %> + + <%= @selected_view_id_name %> + + <%= hidden_input(f, :view_id, readonly: "true", value: @selected_view_id) %>
- <%= styled_label f, :start_date, "From" %> - <%= PlausibleWeb.EmailView.date_format(start_date) %> - <%= hidden_input f, :start_date, value: start_date, readonly: "true" %> + <%= styled_label(f, :start_date, "From") %> + + <%= PlausibleWeb.EmailView.date_format(start_date) %> + + <%= hidden_input(f, :start_date, value: start_date, readonly: "true") %>
→
- <%= styled_label f, :end_date, "To" %> - <%= PlausibleWeb.EmailView.date_format(@end_date) %> - <%= hidden_input f, :end_date, value: @end_date, readonly: "true" %> + <%= styled_label(f, :end_date, "To") %> + + <%= PlausibleWeb.EmailView.date_format(@end_date) %> + + <%= hidden_input(f, :end_date, value: @end_date, readonly: "true") %>
<% {:error, error} -> %> -

The following error occurred when fetching your Google Analytics data.

+

+ The following error occurred when fetching your Google Analytics data. +

<%= error %>

- <% end %> + <% end %> - <%= submit "Confirm import", class: "button mt-6" %> + <%= submit("Confirm import", class: "button mt-6") %> <% end %> diff --git a/lib/plausible_web/templates/universal_analytics/user_metric_form.html.heex b/lib/plausible_web/templates/universal_analytics/user_metric_form.html.heex new file mode 100644 index 0000000000..2f353f984b --- /dev/null +++ b/lib/plausible_web/templates/universal_analytics/user_metric_form.html.heex @@ -0,0 +1,46 @@ +
+

Import from Google Analytics

+ +
+

+ + + + Important: Since your GA property includes data from before 23rd August 2016, you have to take an extra step to make sure we can import data smoothly. +

+ +
    +
  1. 1. Navigate to the GA property you want to import from
    2. +
  2. 2. Go to Admin > Property Settings > User Analysis
    3. +
  3. 3. Make sure Enable Users Metric in Reporting is OFF
    4. +
+ +

+ The setting may take a few minutes to take effect. If your imported data is showing 0 visitors in unexpected places, it's probably caused by this and you + can try importing again later. +

+
+ + <%= link("Continue ->", + to: + Routes.universal_analytics_path(@conn, :confirm, @site.domain, + view_id: @view_id, + access_token: @access_token, + refresh_token: @refresh_token, + expires_at: @expires_at, + legacy: @legacy + ), + class: "button mt-6" + ) %> +
diff --git a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex b/lib/plausible_web/templates/universal_analytics/view_id_form.html.heex similarity index 66% rename from lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex rename to lib/plausible_web/templates/universal_analytics/view_id_form.html.heex index d8b8db4442..e9e371f75c 100644 --- a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex +++ b/lib/plausible_web/templates/universal_analytics/view_id_form.html.heex @@ -1,4 +1,4 @@ -<%= form_for @conn, Routes.site_path(@conn, :import_from_google_view_id, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %> +<%= form_for @conn, Routes.universal_analytics_path(@conn, :view_id, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>

Import from Google Analytics

<%= hidden_input(f, :access_token, value: @access_token) %> @@ -12,9 +12,9 @@
<%= styled_label(f, :view_id, "Google Analytics view") %> - <%= styled_select f, :view_id, @view_ids, prompt: "(Choose view)", required: "true" %> + <%= styled_select(f, :view_id, @view_ids, prompt: "(Choose view)", required: "true") %> <%= styled_error(@conn.assigns[:selected_view_id_error]) %>
- <%= submit "Continue ->", class: "button mt-6" %> + <%= submit("Continue ->", class: "button mt-6") %> <% end %> diff --git a/lib/plausible_web/views/google_analytics4_view.ex b/lib/plausible_web/views/google_analytics4_view.ex new file mode 100644 index 0000000000..760309b804 --- /dev/null +++ b/lib/plausible_web/views/google_analytics4_view.ex @@ -0,0 +1,4 @@ +defmodule PlausibleWeb.GoogleAnalytics4View do + use PlausibleWeb, :view + use Plausible +end diff --git a/lib/plausible_web/views/universal_analytics_view.ex b/lib/plausible_web/views/universal_analytics_view.ex new file mode 100644 index 0000000000..f95899ee77 --- /dev/null +++ b/lib/plausible_web/views/universal_analytics_view.ex @@ -0,0 +1,4 @@ +defmodule PlausibleWeb.UniversalAnalyticsView do + use PlausibleWeb, :view + use Plausible +end diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exs index c448e7c431..076a1af469 100644 --- a/test/plausible/google/api_test.exs +++ b/test/plausible/google/api_test.exs @@ -1,8 +1,8 @@ -defmodule Plausible.Google.ApiTest do +defmodule Plausible.Google.APITest do use Plausible.DataCase, async: true use Plausible.Test.Support.HTTPMocker - alias Plausible.Google.Api + alias Plausible.Google alias Plausible.Imported.UniversalAnalytics import ExUnit.CaptureLog @@ -44,7 +44,7 @@ defmodule Plausible.Google.ApiTest do Plausible.Imported.Buffer.insert_many(buffer, table, records) end - assert :ok == Plausible.Google.Api.import_analytics(date_range, view_id, auth, persist_fn) + assert :ok == Google.UA.API.import_analytics(date_range, view_id, auth, persist_fn) Plausible.Imported.Buffer.flush(buffer) Plausible.Imported.Buffer.stop(buffer) @@ -86,7 +86,7 @@ defmodule Plausible.Google.ApiTest do Plausible.Imported.Buffer.insert_many(buffer, table, records) end - assert :ok == Plausible.Google.Api.import_analytics(range, "123551", auth, persist_fn) + assert :ok == Google.UA.API.import_analytics(range, "123551", auth, persist_fn) Plausible.Imported.Buffer.flush(buffer) Plausible.Imported.Buffer.stop(buffer) @@ -98,7 +98,7 @@ defmodule Plausible.Google.ApiTest do @tag :slow test "will fetch and persist import data from Google Analytics" do - request = %Plausible.Google.ReportRequest{ + request = %Plausible.Google.UA.ReportRequest{ dataset: "imported_exit_pages", view_id: "123", date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]), @@ -127,7 +127,7 @@ defmodule Plausible.Google.ApiTest do hideValueRanges: true, metrics: [%{expression: "ga:users"}, %{expression: "ga:exits"}], orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}], - pageSize: 10000, + pageSize: 10_000, pageToken: nil, viewId: "123" } @@ -139,7 +139,7 @@ defmodule Plausible.Google.ApiTest do ) assert :ok = - Api.fetch_and_persist(request, + Google.UA.API.fetch_and_persist(request, sleep_time: 0, persist_fn: fn dataset, row -> assert dataset == "imported_exit_pages" @@ -167,7 +167,7 @@ defmodule Plausible.Google.ApiTest do end ) - request = %Plausible.Google.ReportRequest{ + request = %Plausible.Google.UA.ReportRequest{ view_id: "123", date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]), dimensions: ["ga:date"], @@ -178,7 +178,7 @@ defmodule Plausible.Google.ApiTest do } assert {:error, :request_failed} = - Api.fetch_and_persist(request, + Google.UA.API.fetch_and_persist(request, sleep_time: 0, persist_fn: fn _dataset, _rows -> :ok end ) @@ -197,7 +197,7 @@ defmodule Plausible.Google.ApiTest do end ) - request = %Plausible.Google.ReportRequest{ + request = %Plausible.Google.UA.ReportRequest{ dataset: "imported_exit_pages", view_id: "123", date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]), @@ -209,7 +209,7 @@ defmodule Plausible.Google.ApiTest do } assert :ok == - Api.fetch_and_persist(request, + Google.UA.API.fetch_and_persist(request, sleep_time: 0, persist_fn: fn dataset, rows -> assert dataset == "imported_exit_pages" @@ -253,7 +253,7 @@ defmodule Plausible.Google.ApiTest do query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} - assert {:error, "google_auth_error"} = Plausible.Google.Api.fetch_stats(site, query, 5) + assert {:error, "google_auth_error"} = Google.API.fetch_stats(site, query, 5) end test "returns whatever error code google returns on API client error", %{site: site} do @@ -270,7 +270,7 @@ defmodule Plausible.Google.ApiTest do query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} - assert {:error, "some_error"} = Plausible.Google.Api.fetch_stats(site, query, 5) + assert {:error, "some_error"} = Google.API.fetch_stats(site, query, 5) end test "returns generic HTTP error and logs it", %{site: site} do @@ -290,7 +290,7 @@ defmodule Plausible.Google.ApiTest do log = capture_log(fn -> assert {:error, "failed_to_list_stats"} = - Plausible.Google.Api.fetch_stats(site, query, 5) + Google.API.fetch_stats(site, query, 5) end) assert log =~ "Google Analytics: failed to list stats: %Finch.Error{reason: :some_reason}" @@ -314,7 +314,7 @@ defmodule Plausible.Google.ApiTest do [ %{name: ["keyword1", "keyword2"], visitors: 25}, %{name: ["keyword3", "keyword4"], visitors: 15} - ]} = Plausible.Google.Api.fetch_stats(site, query, 5) + ]} = Google.API.fetch_stats(site, query, 5) end test "returns next page when page argument is set", %{user: user, site: site} do @@ -336,7 +336,7 @@ defmodule Plausible.Google.ApiTest do [ %{name: ["keyword1", "keyword2"], visitors: 25}, %{name: ["keyword3", "keyword4"], visitors: 15} - ]} = Plausible.Google.Api.fetch_stats(site, query, 5) + ]} = Google.API.fetch_stats(site, query, 5) end test "defaults first page when page argument is not set", %{user: user, site: site} do @@ -355,7 +355,7 @@ defmodule Plausible.Google.ApiTest do [ %{name: ["keyword1", "keyword2"], visitors: 25}, %{name: ["keyword3", "keyword4"], visitors: 15} - ]} = Plausible.Google.Api.fetch_stats(site, query, 5) + ]} = Google.API.fetch_stats(site, query, 5) end test "returns error when token refresh fails", %{user: user, site: site} do @@ -372,7 +372,7 @@ defmodule Plausible.Google.ApiTest do query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])} - assert {:error, "invalid_grant"} = Plausible.Google.Api.fetch_stats(site, query, 5) + assert {:error, "invalid_grant"} = Google.API.fetch_stats(site, query, 5) end end @@ -393,7 +393,7 @@ defmodule Plausible.Google.ApiTest do %{ "one.test" => [{"57238190 - one.test", "57238190"}], "two.test" => [{"54460083 - two.test", "54460083"}] - }} == Plausible.Google.Api.list_views("access_token") + }} == Google.UA.API.list_views("access_token") end test "list_views/1 returns authentication_failed when request fails with HTTP 403" do @@ -405,7 +405,7 @@ defmodule Plausible.Google.ApiTest do end ) - assert {:error, :authentication_failed} == Plausible.Google.Api.list_views("access_token") + assert {:error, :authentication_failed} == Google.UA.API.list_views("access_token") end test "list_views/1 returns authentication_failed when request fails with HTTP 401" do @@ -417,7 +417,7 @@ defmodule Plausible.Google.ApiTest do end ) - assert {:error, :authentication_failed} == Plausible.Google.Api.list_views("access_token") + assert {:error, :authentication_failed} == Google.UA.API.list_views("access_token") end test "list_views/1 returns error when request fails with HTTP 500" do @@ -429,6 +429,6 @@ defmodule Plausible.Google.ApiTest do end ) - assert {:error, :unknown} == Plausible.Google.Api.list_views("access_token") + assert {:error, :unknown} == Google.UA.API.list_views("access_token") end end diff --git a/test/plausible/google/ga4/api_test.exs b/test/plausible/google/ga4/api_test.exs new file mode 100644 index 0000000000..029c46ff7c --- /dev/null +++ b/test/plausible/google/ga4/api_test.exs @@ -0,0 +1,57 @@ +defmodule Plausible.Google.GA4.APITest do + use Plausible.DataCase, async: true + + import Mox + + alias Plausible.Google.GA4 + + setup :verify_on_exit! + + describe "list_properties/1" do + test "returns list of properties grouped by accounts" do + result = Jason.decode!(File.read!("fixture/ga4_list_properties.json")) + + expect(Plausible.HTTPClient.Mock, :get, fn _url, _opts -> + {:ok, %Finch.Response{status: 200, body: result}} + end) + + assert {:ok, accounts} = GA4.API.list_properties("some_access_token") + + assert [ + {"account.one (accounts/28425178)", + [{"account.one - GA4 (properties/428685906)", "properties/428685906"}]}, + {"Demo Account (accounts/54516992)", + [ + {"GA4 - Flood-It! (properties/153293282)", "properties/153293282"}, + {"GA4 - Google Merch Shop (properties/213025502)", "properties/213025502"} + ]} + ] = accounts + end + end + + describe "get_property/2" do + test "returns tuple consisting of display name and value of a property" do + result = Jason.decode!(File.read!("fixture/ga4_list_properties.json")) + + expect(Plausible.HTTPClient.Mock, :get, fn _url, _opts -> + {:ok, %Finch.Response{status: 200, body: result}} + end) + + assert {:ok, {"GA4 - Flood-It! (properties/153293282)", "properties/153293282"}} = + GA4.API.get_property("some_access_token", "properties/153293282") + end + end + + describe "get_analytics_start_date/2" do + test "returns stats start date for a given property" do + result = Jason.decode!(File.read!("fixture/ga4_start_date.json")) + + expect(Plausible.HTTPClient.Mock, :post, fn _url, _headers, _body -> + {:ok, %Finch.Response{status: 200, body: result}} + end) + + assert {:ok, ~D[2024-02-22]} = + GA4.API.get_analytics_start_date("some_access_token", "properties/153293282") + end + end +end diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exs new file mode 100644 index 0000000000..20e0384405 --- /dev/null +++ b/test/plausible/imported/google_analytics4_test.exs @@ -0,0 +1,98 @@ +defmodule Plausible.Imported.GoogleAnalytics4Test do + use Plausible.DataCase, async: true + + import Mox + + import Ecto.Query, only: [from: 2] + + alias Plausible.Imported.GoogleAnalytics4 + + @refresh_token_body Jason.decode!(File.read!("fixture/ga_refresh_token.json")) + + @full_report_mock [ + "fixture/ga4_report_imported_visitors.json", + "fixture/ga4_report_imported_sources.json", + "fixture/ga4_report_imported_pages.json", + "fixture/ga4_report_imported_entry_pages.json", + "fixture/ga4_report_imported_locations.json", + "fixture/ga4_report_imported_devices.json", + "fixture/ga4_report_imported_browsers.json", + "fixture/ga4_report_imported_operating_systems.json" + ] + |> Enum.map(&File.read!/1) + |> Enum.map(&Jason.decode!/1) + + setup :verify_on_exit! + + describe "parse_args/1 and import_data/2" do + setup [:create_user, :create_new_site] + + test "imports data returned from GA4 Data API", %{user: user, site: site} do + past = DateTime.add(DateTime.utc_now(), -3600, :second) + + {:ok, job} = + Plausible.Imported.GoogleAnalytics4.new_import( + site, + user, + property: "properties/123456", + start_date: ~D[2024-02-20], + end_date: Date.utc_today(), + access_token: "redacted_access_token", + refresh_token: "redacted_refresh_token", + token_expires_at: DateTime.to_iso8601(past) + ) + + site_import = Plausible.Imported.get_import(job.args.import_id) + + opts = job |> Repo.reload!() |> Map.get(:args) |> GoogleAnalytics4.parse_args() + + opts = Keyword.put(opts, :flush_interval_ms, 10) + + expect(Plausible.HTTPClient.Mock, :post, fn "https://www.googleapis.com/oauth2/v4/token", + headers, + body -> + assert [{"content-type", "application/x-www-form-urlencoded"}] == headers + + assert %{ + grant_type: :refresh_token, + redirect_uri: "http://localhost:8000/auth/google/callback", + refresh_token: "redacted_refresh_token" + } = body + + {:ok, %Finch.Response{status: 200, body: @refresh_token_body}} + end) + + for report <- @full_report_mock do + expect(Plausible.HTTPClient.Mock, :post, fn _url, headers, _body, _opts -> + assert [{"Authorization", "Bearer 1/fFAGRNJru1FTz70BzhT3Zg"}] == headers + {:ok, %Finch.Response{status: 200, body: report}} + end) + end + + Enum.each(Plausible.Imported.tables(), fn table -> + query = from(imported in table, where: imported.site_id == ^site.id) + assert await_clickhouse_count(query, 0) + end) + + assert :ok = GoogleAnalytics4.import_data(site_import, opts) + + Enum.each(Plausible.Imported.tables(), fn table -> + count = + case table do + "imported_sources" -> 3 + "imported_visitors" -> 3 + "imported_pages" -> 8 + "imported_entry_pages" -> 4 + "imported_exit_pages" -> 0 + "imported_locations" -> 4 + "imported_devices" -> 4 + "imported_browsers" -> 5 + "imported_operating_systems" -> 4 + end + + query = from(imported in table, where: imported.site_id == ^site.id) + assert await_clickhouse_count(query, count) + end) + end + end +end diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index d26395638e..45f1c02381 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -1461,7 +1461,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ]) conn = get(conn, "/api/stats/#{site.domain}/referrers/Google?period=day") - {:ok, terms} = Plausible.Google.Api.Mock.fetch_stats(nil, nil, nil) + {:ok, terms} = Plausible.Google.API.Mock.fetch_stats(nil, nil, nil) assert json_response(conn, 200) == %{ "total_visitors" => 2, diff --git a/test/support/google_api_mock.ex b/test/support/google_api_mock.ex index 5aa33a7180..878cbec970 100644 --- a/test/support/google_api_mock.ex +++ b/test/support/google_api_mock.ex @@ -1,4 +1,8 @@ -defmodule Plausible.Google.Api.Mock do +defmodule Plausible.Google.API.Mock do + @moduledoc """ + Mock of API to Google services. + """ + def fetch_stats(_auth, _query, _limit) do {:ok, [