Map lowercase tagged sources to capitalized form during ingestion (#4417)

* Map lowercase tagged sources to capitalized form during ingestion

* @moduledoc

* Add changlog entry

CHANGELOG.md

@@ -33,6 +33,7 @@ All notable changes to this project will be documented in this file.
 - `is` and `is not` filters in dashboard no longer support wildcards. Use contains/does not contain filter instead.
 - `bounce_rate` metric now returns 0 instead of null for event:page breakdown when page has never been entry page.
 - Make `TOTP_VAULT_KEY` optional plausible/analytics#4317
+- Sources like 'google' and 'facebook' are now stored in capitalized forms ('Google', 'Facebook') plausible/analytics#4417
 
 ### Fixed
 

lib/mix/tasks/send_pageview.ex

@@ -17,6 +17,7 @@ defmodule Mix.Tasks.SendPageview do
   @default_referrer "https://google.com"
   @default_event "pageview"
   @default_props "{}"
+  @default_queryparams ""
   @options [
     ip: :string,
     user_agent: :string,
@@ -28,7 +29,8 @@ defmodule Mix.Tasks.SendPageview do
     event: :string,
     props: :string,
     revenue_currency: :string,
-    revenue_amount: :string
+    revenue_amount: :string,
+    queryparams: :string
   ]
 
   def run(opts) do
@@ -88,6 +90,7 @@ defmodule Mix.Tasks.SendPageview do
     event = Keyword.get(opts, :event, @default_event)
     props = Keyword.get(opts, :props, @default_props)
     hostname = Keyword.get(opts, :hostname, domain)
+    queryparams = Keyword.get(opts, :queryparams, @default_queryparams)
 
     revenue =
       if Keyword.get(opts, :revenue_currency) do
@@ -99,7 +102,7 @@ defmodule Mix.Tasks.SendPageview do
 
     %{
       name: event,
-      url: "http://#{hostname}#{page}",
+      url: "http://#{hostname}#{page}?#{queryparams}",
       domain: domain,
       referrer: referrer,
       props: props,

lib/plausible/application.ex

@@ -105,6 +105,7 @@ defmodule Plausible.Application do
 
     setup_geolocation()
     Location.load_all()
+    Plausible.Ingestion.Acquisition.init()
     Plausible.Geo.await_loader()
 
     Supervisor.start_link(List.flatten(children), opts)

lib/plausible/ingestion/acquisition.ex

@@ -0,0 +1,27 @@
+defmodule Plausible.Ingestion.Acquisition do
+  @moduledoc false
+
+  def init() do
+    :ets.new(__MODULE__, [
+      :named_table,
+      :set,
+      :public,
+      {:read_concurrency, true}
+    ])
+
+    [{"referers.yml", map}] = RefInspector.Database.list(:default)
+
+    Enum.flat_map(map, fn {_, entries} ->
+      Enum.map(entries, fn {_, _, _, _, _, _, name} ->
+        :ets.insert(__MODULE__, {String.downcase(name), name})
+      end)
+    end)
+  end
+
+  def find_mapping(source) do
+    case :ets.lookup(__MODULE__, source) do
+      [{_, name}] -> name
+      _ -> source
+    end
+  end
+end

lib/plausible/ingestion/event.ex

@@ -385,12 +385,16 @@ defmodule Plausible.Ingestion.Event do
   end
 
   defp get_referrer_source(request, ref) do
-    source =
+    tagged_source =
       request.query_params["utm_source"] ||
         request.query_params["source"] ||
         request.query_params["ref"]
 
-    source || PlausibleWeb.RefInspector.parse(ref)
+    if tagged_source do
+      Plausible.Ingestion.Acquisition.find_mapping(tagged_source)
+    else
+      PlausibleWeb.RefInspector.parse(ref)
+    end
   end
 
   defp clean_referrer(nil), do: nil

test/plausible_web/controllers/api/external_controller_test.exs

@@ -239,7 +239,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
       assert event.browser_version == "70.0"
     end
 
-    test "parses referrer", %{conn: conn, site: site} do
+    test "parses referrer source", %{conn: conn, site: site} do
       params = %{
         name: "pageview",
         url: "http://example.com/",
@@ -407,6 +407,22 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
       assert session.referrer_source == "betalist"
     end
 
+    test "if utm_source matches a capitalized form from ref_inspector, the capitalized form is recorded",
+         %{conn: conn, site: site} do
+      params = %{
+        name: "pageview",
+        url: "http://www.example.com/?utm_source=facebook",
+        domain: site.domain
+      }
+
+      conn
+      |> post("/api/event", params)
+
+      session = get_created_session(site)
+
+      assert session.referrer_source == "Facebook"
+    end
+
     test "utm tags are stored", %{conn: conn, site: site} do
       params = %{
         name: "pageview",