Detector.js - detect v1 and technologies used on the website (#5591)

* fix comment on localhost dogfood tracking

* improve detector script and integrate into Elixir

* wait for window.plausible.l instead of window.plausible

* do not touch source files during compilation

* stop referencing compiler hint module attr

extra/lib/plausible_web/dogfood.ex

@@ -34,10 +34,13 @@ defmodule PlausibleWeb.Dogfood do
         env in ["dev", "ce_dev"] ->
           # By default we're not letting the app track itself on localhost.
           # The requested script will be `s-.js` and it will respond with 404.
-          # If you wish to track the app itself, uncomment the following line
+          # If you wish to track the app itself, uncomment the following code
           # and replace the site_id if necessary (1 stands for dummy.site).
 
-          # PlausibleWeb.Tracker.get_or_create_tracker_script_configuration!(1).id
+          # Plausible.Repo.get(Plausible.Site, 1)
+          # |> PlausibleWeb.Tracker.get_or_create_tracker_script_configuration!()
+          # |> Map.get(:id)
+
           ""
 
         env in ["test", "ce_test"] ->

lib/plausible/installation_support/checks/installation.ex

@@ -1,12 +1,14 @@
 defmodule Plausible.InstallationSupport.Checks.Installation do
   require Logger
 
-  path = Application.app_dir(:plausible, "priv/tracker/installation_support/verifier-v1.js")
-  # On CI, the file might not be present for static checks so we create an empty one
-  File.touch!(path)
+  @verifier_code_path "priv/tracker/installation_support/verifier-v1.js"
+  @external_resource @verifier_code_path
 
-  @verifier_code File.read!(path)
-  @external_resource "priv/tracker/installation_support/verifier-v1.js"
+  # On CI, the file might not be present for static checks so we default to empty string
+  @verifier_code (case File.read(Application.app_dir(:plausible, @verifier_code_path)) do
+                    {:ok, content} -> content
+                    {:error, _} -> ""
+                  end)
 
   # Puppeteer wrapper function that executes the vanilla JS verifier code
   @puppeteer_wrapper_code """

lib/plausible/installation_support/detection.ex

@@ -0,0 +1,88 @@
+defmodule Plausible.InstallationSupport.Detection do
+  @moduledoc """
+  Exposes a perform function which visits the given URL via a Browserless
+  /function API call, and in a returns the following diagnostics:
+
+  * v1_detected (optional - detection can take up to 3s)
+  * gtm_likely
+  * wordpress_likely
+  * wordpress_plugin
+
+  These diagnostics are used to determine what installation type to recommend,
+  and whether to provide a notice for upgrading an existing v1 integration to v2.
+  """
+  require Logger
+  alias Plausible.InstallationSupport
+
+  @detector_code_path "priv/tracker/installation_support/detector.js"
+  @external_resource @detector_code_path
+
+  # On CI, the file might not be present for static checks so we default to empty string
+  @detector_code (case File.read(Application.app_dir(:plausible, @detector_code_path)) do
+                    {:ok, content} -> content
+                    {:error, _} -> ""
+                  end)
+
+  # Puppeteer wrapper function that executes the vanilla JS verifier code
+  @puppeteer_wrapper_code """
+  export default async function({ page, context }) {
+    try {
+      await page.setUserAgent(context.userAgent);
+      await page.goto(context.url);
+
+      await page.evaluate(() => {
+        #{@detector_code}
+      });
+
+      return await page.evaluate(async (detectV1, debug) => {
+        return await window.scanPageBeforePlausibleInstallation(detectV1, debug);
+      }, context.detectV1, context.debug);
+    } catch (error) {
+      const msg = error.message ? error.message : JSON.stringify(error)
+      return {data: {completed: false, error: msg}}
+    }
+  }
+  """
+
+  def perform(url, opts \\ []) do
+    req_opts =
+      [
+        headers: %{content_type: "application/json"},
+        body:
+          Jason.encode!(%{
+            code: @puppeteer_wrapper_code,
+            context: %{
+              url: url,
+              userAgent: InstallationSupport.user_agent(),
+              detectV1: Keyword.get(opts, :detect_v1?, false),
+              debug: Application.get_env(:plausible, :environment) == "dev"
+            }
+          }),
+        retry: :transient,
+        retry_log_level: :warning,
+        max_retries: 2
+      ]
+      |> Keyword.merge(Application.get_env(:plausible, __MODULE__)[:req_opts] || [])
+
+    case Req.post(InstallationSupport.browserless_function_api_endpoint(), req_opts) do
+      {:ok, %{status: 200, body: %{"data" => %{"completed" => true} = js_data}}} ->
+        {:ok,
+         %{
+           v1_detected: js_data["v1Detected"],
+           gtm_likely: js_data["gtmLikely"],
+           wordpress_likely: js_data["wordpressLikely"],
+           wordpress_plugin: js_data["wordpressPlugin"]
+         }}
+
+      {:ok, %{body: %{"data" => %{"error" => error}}}} ->
+        Logger.warning("[DETECTION] Browserless JS error (url='#{url}'): #{inspect(error)}")
+
+        {:error, {:browserless, error}}
+
+      {:error, %{reason: reason}} ->
+        Logger.warning("[DETECTION] Browserless request error (url='#{url}'): #{inspect(reason)}")
+
+        {:error, {:req, reason}}
+    end
+  end
+end

tracker/installation_support/detector.js

@@ -1,4 +1,4 @@
-import { waitForSnippetsV1 } from "./snippet-checks"
+import { waitForPlausibleFunction } from "./plausible-function-check"
 import { checkWordPress } from "./check-wordpress"
 import { checkGTM } from "./check-gtm"
 
@@ -7,6 +7,16 @@ window.scanPageBeforePlausibleInstallation = async function(detectV1, debug) {
     if (debug) console.log('[Plausible Verification]', message)
   }
 
+  let v1Detected = null
+
+  if (detectV1) {
+    log('Waiting for Plausible function...')
+    const plausibleFound = await waitForPlausibleFunction(3000)
+    log(`plausibleFound: ${plausibleFound}`)
+    v1Detected = plausibleFound && typeof window.plausible.s === 'undefined'
+    log(`v1Detected: ${v1Detected}`)
+  }
+
   const {wordpressPlugin, wordpressLikely} = checkWordPress(document)
   log(`wordpressPlugin: ${wordpressPlugin}`)
   log(`wordpressLikely: ${wordpressLikely}`)
@@ -14,15 +24,6 @@ window.scanPageBeforePlausibleInstallation = async function(detectV1, debug) {
   const gtmLikely = checkGTM(document)
   log(`gtmLikely: ${gtmLikely}`)
 
-  // Cannot implement yet: we should detect the WP plugin version here and
-  // decide `v1Detected` based on that. For now we assume WP plugin is v1.
-  let v1Detected = wordpressPlugin
-
-  if (!v1Detected && detectV1) {
-    const snippetData = await waitForSnippetsV1(log)
-    v1Detected = snippetData.counts.all > 0
-  }
-
   return {
     data: {
       completed: true,

tracker/installation_support/plausible-function-check.js

@@ -15,13 +15,13 @@ export async function plausibleFunctionCheck(log) {
   }
 }
 
-async function waitForPlausibleFunction() {
+export async function waitForPlausibleFunction(timeout = 5000) {
   const checkFn = (opts) => {
-    if (window.plausible) { return true }
+    if (window.plausible?.l) { return true }
     if (opts.timeout) { return false }
     return 'continue'
   }
-  return await runThrottledCheck(checkFn, {timeout: 5000, interval: 100})
+  return await runThrottledCheck(checkFn, {timeout: timeout, interval: 100})
 }
 
 function testPlausibleCallback(log) {

tracker/test/installation_support/detector.spec.js

@@ -2,7 +2,7 @@ import { test, expect } from '@playwright/test'
 import { detect } from '../support/installation-support-playwright-wrappers'
 import { initializePageDynamically } from '../support/initialize-page-dynamically'
 
-test.describe('detector.js (basic diagnostics)', () => {
+test.describe('detector.js (tech recognition)', () => {
   test('skips v1 snippet detection by default', async ({ page }, { testId }) => {
     const { url } = await initializePageDynamically(page, {
       testId,
@@ -17,7 +17,7 @@ test.describe('detector.js (basic diagnostics)', () => {
 
     const result = await detect(page, {url: url, detectV1: false})
 
-    expect(result.data.v1Detected).toBe(false)
+    expect(result.data.v1Detected).toBe(null)
   })
 
   test('detects WP plugin, WP and GTM', async ({ page }, { testId }) => {
@@ -54,3 +54,56 @@ test.describe('detector.js (basic diagnostics)', () => {
     expect(result.data.gtmLikely).toBe(false)
   })
 })
+
+test.describe('detector.js (v1 detection)', () => {
+  test('v1Detected is true when v1 plausible exists + detects WP plugin, WP and GTM', async ({ page }, { testId }) => {
+    const { url } = await initializePageDynamically(page, {
+      testId,
+      response: `
+        <html>
+          <head>
+            <link rel="icon" href="https://example.com/wp-content/uploads/favicon.ico" sizes="32x32">
+            <meta name="plausible-analytics-version" content="2.3.1">
+            <script async src="https://www.googletagmanager.com/gtm.js?id=GTM-123"></script>
+            <script defer src="/tracker/js/plausible.local.manual.js" data-domain="abc.de"></script>
+          </head>
+        </html>
+      `
+    })
+
+    const result = await detect(page, {url: url, detectV1: true})
+
+    expect(result.data.v1Detected).toBe(true)
+    expect(result.data.wordpressPlugin).toBe(true)
+    expect(result.data.wordpressLikely).toBe(true)
+    expect(result.data.gtmLikely).toBe(true)
+  })
+
+  test('v1Detected is false when plausible function does not exist', async ({ page }, { testId }) => {
+    const { url } = await initializePageDynamically(page, {
+      testId,
+      response: '<html><head></head></html>'
+    })
+
+    const result = await detect(page, {url: url, detectV1: true})
+
+    expect(result.data.v1Detected).toBe(false)
+    expect(result.data.wordpressPlugin).toBe(false)
+    expect(result.data.wordpressLikely).toBe(false)
+    expect(result.data.gtmLikely).toBe(false)
+  })
+
+  test('v1Detected is false when v2 plausible installed', async ({ page }, { testId }) => {
+    const { url } = await initializePageDynamically(page, {
+      testId,
+      scriptConfig: {
+        domain: 'abc.de',
+        captureOnLocalhost: false
+      }
+    })
+
+    const result = await detect(page, {url: url, detectV1: true})
+
+    expect(result.data.v1Detected).toBe(false)
+  })
+})