Check for Sites API feature against respective team when using API key (#5753)

* Account for feature check error when adding custom prop via Sites API

* Expand Teams API with team membership check predicate

* Validate feature availability for Sites API endpoints

* Refactor tests to account for differrent errors

extra/lib/plausible_web/controllers/api/external_sites_controller.ex

@@ -483,6 +483,12 @@ defmodule PlausibleWeb.Api.ExternalSitesController do
       {:missing, param} ->
         H.bad_request(conn, "Parameter `#{param}` is required to create a custom property")
 
+      {:error, :upgrade_required} ->
+        H.payment_required(
+          conn,
+          "Your current subscription plan does not include Custom Properties"
+        )
+
       {:error, changeset} ->
         %{allowed_event_props: [error | _]} =
           Ecto.Changeset.traverse_errors(changeset, fn {_msg, opts} ->

lib/plausible/teams/memberships.ex

@@ -31,6 +31,8 @@ defmodule Plausible.Teams.Memberships do
     )
   end
 
+  @spec team_role(Teams.Team.t(), Auth.User.t()) ::
+          {:ok, Teams.Membership.role()} | {:error, :not_a_member}
   def team_role(team, user) do
     result =
       from(u in Auth.User,
@@ -86,6 +88,7 @@ defmodule Plausible.Teams.Memberships do
     end
   end
 
+  @spec site_member?(Plausible.Site.t(), Auth.User.t() | nil) :: boolean()
   def site_member?(site, user) do
     case site_role(site, user) do
       {:ok, _} -> true
@@ -93,6 +96,14 @@ defmodule Plausible.Teams.Memberships do
     end
   end
 
+  @spec team_member?(Teams.Team.t(), Auth.User.t()) :: boolean()
+  def team_member?(team, user) do
+    case team_role(team, user) do
+      {:ok, _} -> true
+      _ -> false
+    end
+  end
+
   @spec has_editor_access?(Plausible.Site.t(), Auth.User.t() | nil) :: boolean()
   def has_editor_access?(site, user) do
     case site_role(site, user) do

lib/plausible_web/plugs/authorize_public_api.ex

@@ -122,13 +122,30 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI do
   defp verify_by_scope(conn, api_key, "stats:read:" <> _ = scope) do
     with :ok <- check_scope(api_key, scope),
          {:ok, site} <- find_site(conn.params["site_id"], api_key),
-         :ok <- verify_site_access(api_key, site) do
+         :ok <- verify_site_access(api_key, site, Plausible.Billing.Feature.StatsAPI) do
       Plausible.OpenTelemetry.add_site_attributes(site)
       site = Plausible.Repo.preload(site, :completed_imports)
       {:ok, assign(conn, :site, site)}
     end
   end
 
+  defp verify_by_scope(conn, api_key, "sites:" <> scope_suffix = scope) do
+    feature =
+      case scope_suffix do
+        "read:" <> _ ->
+          Plausible.Billing.Feature.StatsAPI
+
+        "provision:" <> _ ->
+          Plausible.Billing.Feature.SitesAPI
+      end
+
+    with :ok <- check_scope(api_key, scope),
+         :ok <- maybe_verify_site_access(conn, api_key, feature),
+         :ok <- maybe_verify_team_access(conn, api_key, feature) do
+      {:ok, conn}
+    end
+  end
+
   defp verify_by_scope(conn, api_key, scope) do
     with :ok <- check_scope(api_key, scope) do
       {:ok, conn}
@@ -173,6 +190,26 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI do
     end
   end
 
+  defp maybe_verify_site_access(conn, api_key, feature) do
+    case find_site(conn.params["site_id"], api_key) do
+      {:ok, site} ->
+        verify_site_access(api_key, site, feature)
+
+      _ ->
+        :ok
+    end
+  end
+
+  defp maybe_verify_team_access(conn, api_key, feature) do
+    team = api_key.team || Teams.get(conn.params["team_id"])
+
+    if team do
+      verify_team_access(api_key, team, feature)
+    else
+      :ok
+    end
+  end
+
   defp find_site(nil, _api_key), do: {:error, :missing_site_id}
 
   defp find_site("rollup:" <> team_identifier, api_key) do
@@ -198,7 +235,7 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI do
     end
   end
 
-  defp verify_site_access(api_key, site) do
+  defp verify_site_access(api_key, site, feature) do
     team = Repo.preload(site, :team).team
 
     is_member? = Plausible.Teams.Memberships.site_member?(site, api_key.user)
@@ -214,7 +251,32 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI do
       Teams.locked?(team) ->
         {:error, :site_locked}
 
-      Plausible.Billing.Feature.StatsAPI.check_availability(team) !== :ok ->
+      feature.check_availability(team) !== :ok ->
+        {:error, :upgrade_required}
+
+      is_member? ->
+        :ok
+
+      true ->
+        {:error, :invalid_api_key}
+    end
+  end
+
+  defp verify_team_access(api_key, team, feature) do
+    is_member? = Plausible.Teams.Memberships.team_member?(team, api_key.user)
+    is_super_admin? = Auth.is_super_admin?(api_key.user_id)
+
+    cond do
+      is_super_admin? ->
+        :ok
+
+      api_key.team_id && api_key.team_id != team.id ->
+        {:error, :invalid_api_key}
+
+      Teams.locked?(team) ->
+        {:error, :site_locked}
+
+      feature.check_availability(team) !== :ok ->
         {:error, :upgrade_required}
 
       is_member? ->
@@ -260,11 +322,17 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI do
     )
   end
 
-  defp send_error(conn, _, {:error, :upgrade_required}) do
+  defp send_error(conn, scope, {:error, :upgrade_required}) do
-    H.payment_required(
+    feature =
-      conn,
+      case scope do
-      "The account that owns this API key does not have access to Stats API. Please make sure you're using the API key of a subscriber account and that the subscription plan includes Stats API"
+        "sites:provision:" <> _ ->
-    )
+          Plausible.Billing.Feature.SitesAPI
+
+        _ ->
+          Plausible.Billing.Feature.StatsAPI
+      end
+
+    feature_payment_error(conn, feature)
   end
 
   defp send_error(conn, _, {:error, :site_locked}) do
@@ -273,4 +341,13 @@ defmodule PlausibleWeb.Plugs.AuthorizePublicAPI do
       "This Plausible site is locked due to missing active subscription. In order to access it, the site owner should subscribe to a suitable plan"
     )
   end
+
+  defp feature_payment_error(conn, feature) do
+    feature_name = feature.display_name()
+
+    H.payment_required(
+      conn,
+      "The account that owns this API key does not have access to #{feature_name}. Please make sure you're using the API key of a subscriber account and that the subscription plan includes #{feature_name}"
+    )
+  end
 end

test/plausible_web/controllers/api/external_sites_controller_sites_crud_api_test.exs

@@ -21,6 +21,17 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerSitesCrudApiTest do
     end
 
     describe "POST /api/v1/sites" do
+      setup %{user: user} do
+        subscribe_to_enterprise_plan(user,
+          features: [
+            Plausible.Billing.Feature.StatsAPI,
+            Plausible.Billing.Feature.SitesAPI
+          ]
+        )
+
+        :ok
+      end
+
       test "can create a site", %{conn: conn} do
         conn =
           post(conn, "/api/v1/sites", %{
@@ -113,15 +124,19 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerSitesCrudApiTest do
             "timezone" => "Europe/Tallinn"
           })
 
-        assert json_response(conn, 403) == %{
+        assert %{"error" => error} = json_response(conn, 402)
-                 "error" => "You can't add sites to the selected team."
+        assert error =~ "API key does not have access to Sites API"
-               }
       end
 
       test "can create a site under a specific team if permitted", %{conn: conn, user: user} do
         _site = new_site(owner: user)
 
-        owner = new_user() |> subscribe_to_growth_plan()
+        owner =
+          new_user()
+          |> subscribe_to_enterprise_plan(
+            features: [Plausible.Billing.Feature.StatsAPI, Plausible.Billing.Feature.SitesAPI]
+          )
+
         team = owner |> team_of() |> Plausible.Teams.complete_setup()
         add_member(team, user: user, role: :owner)
 
@@ -159,7 +174,13 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerSitesCrudApiTest do
       test "creates under a particular team when team-scoped key used", %{conn: conn, user: user} do
         personal_team = user |> subscribe_to_business_plan() |> team_of()
 
-        another_team = new_user() |> subscribe_to_business_plan() |> team_of()
+        another_team =
+          new_user()
+          |> subscribe_to_enterprise_plan(
+            features: [Plausible.Billing.Feature.StatsAPI, Plausible.Billing.Feature.SitesAPI]
+          )
+          |> team_of()
+
         add_member(another_team, user: user, role: :admin)
 
         api_key = insert(:api_key, user: user, team: another_team, scopes: ["sites:provision:*"])
@@ -331,21 +352,6 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerSitesCrudApiTest do
                        }) = response
       end
 
-      test "does not allow creating more sites than the limit", %{conn: conn, user: user} do
-        for _ <- 1..10, do: new_site(owner: user)
-
-        conn =
-          post(conn, "/api/v1/sites", %{
-            "domain" => "some-site.domain",
-            "timezone" => "Europe/Tallinn"
-          })
-
-        assert json_response(conn, 402) == %{
-                 "error" =>
-                   "Your account has reached the limit of 10 sites. To unlock more sites, please upgrade your subscription."
-               }
-      end
-
       test "cannot access with a bad API key scope", %{conn: conn, user: user} do
         api_key = insert(:api_key, user: user, scopes: ["stats:read:*"])
 
@@ -364,6 +370,17 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerSitesCrudApiTest do
     describe "DELETE /api/v1/sites/:site_id" do
       setup :create_site
 
+      setup %{user: user} do
+        subscribe_to_enterprise_plan(user,
+          features: [
+            Plausible.Billing.Feature.StatsAPI,
+            Plausible.Billing.Feature.SitesAPI
+          ]
+        )
+
+        :ok
+      end
+
       test "delete a site by its domain", %{conn: conn, site: site} do
         conn = delete(conn, "/api/v1/sites/" <> site.domain)
 
@@ -393,7 +410,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerSitesCrudApiTest do
         add_guest(site, user: user, role: :editor)
         conn = delete(conn, "/api/v1/sites/" <> site.domain)
 
-        assert json_response(conn, 404) == %{"error" => "Site could not be found"}
+        assert %{"error" => error} = json_response(conn, 402)
+        assert error =~ "API key does not have access to Sites API"
       end
 
       test "cannot delete if team not matching team-scoped API key", %{
@@ -408,7 +426,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerSitesCrudApiTest do
 
         conn = delete(conn, "/api/v1/sites/" <> site.domain)
 
-        assert json_response(conn, 404) == %{"error" => "Site could not be found"}
+        assert %{"error" => error} = json_response(conn, 401)
+        assert error =~ "Invalid API key"
       end
 
       test "cannot access with a bad API key scope", %{conn: conn, site: site, user: user} do
@@ -669,8 +688,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerSitesCrudApiTest do
 
         conn = get(conn, "/api/v1/sites/" <> site.domain)
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 401)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "Invalid API key"
       end
 
       test "is 404 when site cannot be found", %{conn: conn} do
@@ -679,18 +698,31 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerSitesCrudApiTest do
         assert json_response(conn, 404) == %{"error" => "Site could not be found"}
       end
 
+      @tag :capture_log
       test "is 404 when user is not a member of the site", %{conn: conn} do
-        site = insert(:site)
+        site = new_site()
 
         conn = get(conn, "/api/v1/sites/" <> site.domain)
 
-        assert json_response(conn, 404) == %{"error" => "Site could not be found"}
+        assert %{"error" => error} = json_response(conn, 401)
+        assert error =~ "Invalid API key"
       end
     end
 
     describe "PUT /api/v1/sites/:site_id" do
       setup :create_site
 
+      setup %{user: user} do
+        subscribe_to_enterprise_plan(user,
+          features: [
+            Plausible.Billing.Feature.StatsAPI,
+            Plausible.Billing.Feature.SitesAPI
+          ]
+        )
+
+        :ok
+      end
+
       test "can change domain name", %{conn: conn, site: site} do
         old_domain = site.domain
         assert old_domain != "new.example.com"
@@ -836,9 +868,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerSitesCrudApiTest do
             "domain" => "new.example.com"
           })
 
-        assert json_response(conn, 404) == %{
+        assert %{"error" => error} = json_response(conn, 401)
-                 "error" => "Site could not be found"
+        assert error =~ "Invalid API key"
-               }
       end
 
       test "fails when neither 'domain' nor 'tracker_script_configuration' is provided", %{

test/plausible_web/controllers/api/external_sites_controller_test.exs

@@ -132,15 +132,20 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
             "timezone" => "Europe/Tallinn"
           })
 
-        assert json_response(conn, 403) == %{
+        assert %{"error" => error} = json_response(conn, 402)
-                 "error" => "You can't add sites to the selected team."
+
-               }
+        assert error =~ "API key does not have access to Sites API"
       end
 
       test "can create a site under a specific team if permitted", %{conn: conn, user: user} do
         _site = new_site(owner: user)
 
-        owner = new_user() |> subscribe_to_growth_plan()
+        owner =
+          new_user()
+          |> subscribe_to_enterprise_plan(
+            features: [Plausible.Billing.Feature.StatsAPI, Plausible.Billing.Feature.SitesAPI]
+          )
+
         team = owner |> team_of() |> Plausible.Teams.complete_setup()
         add_member(team, user: user, role: :owner)
 
@@ -163,7 +168,13 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
       test "creates under a particular team when team-scoped key used", %{conn: conn, user: user} do
         personal_team = user |> subscribe_to_business_plan() |> team_of()
 
-        another_team = new_user() |> subscribe_to_business_plan() |> team_of()
+        another_team =
+          new_user()
+          |> subscribe_to_enterprise_plan(
+            features: [Plausible.Billing.Feature.StatsAPI, Plausible.Billing.Feature.SitesAPI]
+          )
+          |> team_of()
+
         add_member(another_team, user: user, role: :admin)
 
         api_key = insert(:api_key, user: user, team: another_team, scopes: ["sites:provision:*"])
@@ -270,6 +281,14 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
     describe "DELETE /api/v1/sites/:site_id" do
       setup :create_site
 
+      setup %{user: user} do
+        subscribe_to_enterprise_plan(user,
+          features: [Plausible.Billing.Feature.StatsAPI, Plausible.Billing.Feature.SitesAPI]
+        )
+
+        :ok
+      end
+
       test "delete a site by its domain", %{conn: conn, site: site} do
         conn = delete(conn, "/api/v1/sites/" <> site.domain)
 
@@ -299,7 +318,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
         add_guest(site, user: user, role: :editor)
         conn = delete(conn, "/api/v1/sites/" <> site.domain)
 
-        assert json_response(conn, 404) == %{"error" => "Site could not be found"}
+        assert %{"error" => error} = json_response(conn, 402)
+        assert error =~ "API key does not have access to Sites API"
       end
 
       test "cannot delete if team not matching team-scoped API key", %{
@@ -314,7 +334,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
 
         conn = delete(conn, "/api/v1/sites/" <> site.domain)
 
-        assert json_response(conn, 404) == %{"error" => "Site could not be found"}
+        assert %{"error" => error} = json_response(conn, 401)
+        assert error =~ "Invalid API key"
       end
 
       test "cannot access with a bad API key scope", %{conn: conn, site: site, user: user} do
@@ -335,6 +356,18 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
     describe "PUT /api/v1/sites/shared-links" do
       setup :create_site
 
+      setup %{user: user} do
+        subscribe_to_enterprise_plan(user,
+          features: [
+            Plausible.Billing.Feature.SharedLinks,
+            Plausible.Billing.Feature.StatsAPI,
+            Plausible.Billing.Feature.SitesAPI
+          ]
+        )
+
+        :ok
+      end
+
       test "can add a shared link to a site", %{conn: conn, site: site} do
         conn =
           put(conn, "/api/v1/sites/shared-links", %{
@@ -397,8 +430,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
             name: "WordPress"
           })
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 401)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "Invalid API key"
       end
 
       test "returns 400 when site id missing", %{conn: conn} do
@@ -437,8 +470,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
             name: "WordPress"
           })
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 402)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "API key does not have access to Sites API"
       end
 
       test "fails to create without access to SharedLinks feature", %{
@@ -478,6 +511,18 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
     describe "PUT /api/v1/sites/custom-props" do
       setup :create_site
 
+      setup %{user: user} do
+        subscribe_to_enterprise_plan(user,
+          features: [
+            Plausible.Billing.Feature.Props,
+            Plausible.Billing.Feature.StatsAPI,
+            Plausible.Billing.Feature.SitesAPI
+          ]
+        )
+
+        :ok
+      end
+
       test "can add a custom property to a site", %{conn: conn, site: site} do
         conn =
           put(conn, "/api/v1/sites/custom-props", %{
@@ -573,8 +618,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
             property: "prop1"
           })
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 401)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "Invalid API key"
       end
 
       test "returns 400 when site id missing", %{conn: conn} do
@@ -613,8 +658,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
             property: "prop1"
           })
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 402)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "API key does not have access to Sites API"
       end
 
       test "returns 400 when property missing", %{conn: conn, site: site} do
@@ -631,6 +676,14 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
     describe "PUT /api/v1/sites/goals" do
       setup :create_site
 
+      setup %{user: user} do
+        subscribe_to_enterprise_plan(user,
+          features: [Plausible.Billing.Feature.StatsAPI, Plausible.Billing.Feature.SitesAPI]
+        )
+
+        :ok
+      end
+
       test "can add a goal as event to a site", %{conn: conn, site: site} do
         conn =
           put(conn, "/api/v1/sites/goals", %{
@@ -747,8 +800,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
             event_name: "Signup"
           })
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 401)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "Invalid API key"
       end
 
       test "returns 400 when site id missing", %{conn: conn} do
@@ -790,8 +843,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
             event_name: "Signup"
           })
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 402)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "API key does not have access to Sites API"
       end
 
       test "returns 400 when goal type missing", %{conn: conn, site: site} do
@@ -831,6 +884,18 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
     describe "DELETE /api/v1/sites/custom-props/:property" do
       setup :create_site
 
+      setup %{user: user} do
+        subscribe_to_enterprise_plan(user,
+          features: [
+            Plausible.Billing.Feature.Props,
+            Plausible.Billing.Feature.StatsAPI,
+            Plausible.Billing.Feature.SitesAPI
+          ]
+        )
+
+        :ok
+      end
+
       test "deletes a custom property", %{conn: conn, site: site} do
         conn =
           put(conn, "/api/v1/sites/custom-props", %{
@@ -914,8 +979,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
             site_id: site.domain
           })
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 401)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "Invalid API key"
       end
 
       test "handles non-existent custom prop gracefully", %{conn: conn, site: site} do
@@ -940,7 +1005,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
             site_id: site.domain
           })
 
-        assert json_response(conn, 404) == %{"error" => "Site could not be found"}
+        assert %{"error" => error} = json_response(conn, 402)
+        assert error =~ "API key does not have access to Sites API"
       end
 
       test "cannot access with a bad API key scope", %{conn: conn, site: site, user: user} do
@@ -966,6 +1032,17 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
     describe "DELETE /api/v1/sites/goals/:goal_id" do
       setup :create_site
 
+      setup %{user: user} do
+        subscribe_to_enterprise_plan(user,
+          features: [
+            Plausible.Billing.Feature.StatsAPI,
+            Plausible.Billing.Feature.SitesAPI
+          ]
+        )
+
+        :ok
+      end
+
       test "delete a goal by its id", %{conn: conn, site: site} do
         conn =
           put(conn, "/api/v1/sites/goals", %{
@@ -1027,8 +1104,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
             site_id: site.domain
           })
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 401)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "Invalid API key"
       end
 
       test "is 404 when goal cannot be found", %{conn: conn, site: site} do
@@ -1053,7 +1130,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
             site_id: site.domain
           })
 
-        assert json_response(conn, 404) == %{"error" => "Site could not be found"}
+        assert %{"error" => error} = json_response(conn, 402)
+        assert error =~ "API key does not have access to Sites API"
       end
 
       test "cannot access with a bad API key scope", %{conn: conn, site: site, user: user} do
@@ -1318,12 +1396,23 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
 
         conn = get(conn, "/api/v1/sites/guests?site_id=#{site.domain}")
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 401)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "Invalid API key"
       end
     end
 
     describe "PUT /api/v1/sites/guests" do
+      setup %{user: user} do
+        subscribe_to_enterprise_plan(user,
+          features: [
+            Plausible.Billing.Feature.StatsAPI,
+            Plausible.Billing.Feature.SitesAPI
+          ]
+        )
+
+        :ok
+      end
+
       test "creates new invitation", %{conn: conn, user: user} do
         site = new_site(owner: user)
 
@@ -1408,8 +1497,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
             "email" => "test@example.com"
           })
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 401)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "Invalid API key"
       end
 
       test "fails for unknown role", %{conn: conn, user: user} do
@@ -1431,6 +1520,14 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
     end
 
     describe "DELETE /api/v1/sites/guests" do
+      setup %{user: user} do
+        subscribe_to_enterprise_plan(user,
+          features: [Plausible.Billing.Feature.StatsAPI, Plausible.Billing.Feature.SitesAPI]
+        )
+
+        :ok
+      end
+
       test "no-op when nothing to delete", %{conn: conn, user: user} do
         site = new_site(owner: user)
 
@@ -1492,8 +1589,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
 
         conn = delete(conn, "/api/v1/sites/guests/test@example.com?site_id=#{site.domain}")
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 401)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "Invalid API key"
       end
 
       test "won't delete non-guest membership", %{conn: conn, user: user} do
@@ -1565,8 +1662,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
 
         conn = get(conn, "/api/v1/sites/" <> site.domain)
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 401)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "Invalid API key"
       end
 
       test "is 404 when site cannot be found", %{conn: conn} do
@@ -1575,12 +1672,14 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
         assert json_response(conn, 404) == %{"error" => "Site could not be found"}
       end
 
-      test "is 404 when user is not a member of the site", %{conn: conn} do
+      @tag :capture_log
-        site = insert(:site)
+      test "is 401 when user is not a member of the site", %{conn: conn} do
+        site = new_site()
 
         conn = get(conn, "/api/v1/sites/" <> site.domain)
 
-        assert json_response(conn, 404) == %{"error" => "Site could not be found"}
+        assert %{"error" => error} = json_response(conn, 401)
+        assert error =~ "Invalid API key"
       end
     end
 
@@ -1650,8 +1749,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
 
         conn = get(conn, "/api/v1/sites/custom-props?site_id=" <> site.domain)
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 401)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "Invalid API key"
       end
 
       test "returns error when `site_id` parameter is missing", %{conn: conn} do
@@ -1663,21 +1762,21 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
       end
 
       test "returns error when `site_id` parameter is invalid", %{conn: conn} do
-        conn = get(conn, "/api/v1/sites/custom-props=does.not.exist")
+        conn = get(conn, "/api/v1/sites/custom-props?site_id=does.not.exist")
 
         assert json_response(conn, 404) == %{
                  "error" => "Site could not be found"
                }
       end
 
+      @tag :capture_log
       test "returns error when user is not a member of the site", %{conn: conn} do
-        site = insert(:site)
+        site = new_site()
 
-        conn = get(conn, "/api/v1/sites/custom-props=" <> site.domain)
+        conn = get(conn, "/api/v1/sites/custom-props?site_id=" <> site.domain)
 
-        assert json_response(conn, 404) == %{
+        assert %{"error" => error} = json_response(conn, 401)
-                 "error" => "Site could not be found"
+        assert(error =~ "Invalid API key")
-               }
       end
     end
 
@@ -1809,8 +1908,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
 
         conn = get(conn, "/api/v1/sites/goals?site_id=" <> site.domain)
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 401)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "Invalid API key"
       end
 
       test "returns error when `site_id` parameter is missing", %{conn: conn} do
@@ -1829,20 +1928,28 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
                }
       end
 
+      @tag :capture_log
       test "returns error when user is not a member of the site", %{conn: conn} do
-        site = insert(:site)
+        site = new_site()
 
         conn = get(conn, "/api/v1/sites/goals?site_id=" <> site.domain)
 
-        assert json_response(conn, 404) == %{
+        assert %{"error" => error} = json_response(conn, 401)
-                 "error" => "Site could not be found"
+        assert error =~ "Invalid API key"
-               }
       end
     end
 
     describe "PUT /api/v1/sites/:site_id" do
       setup :create_site
 
+      setup %{user: user} do
+        subscribe_to_enterprise_plan(user,
+          features: [Plausible.Billing.Feature.StatsAPI, Plausible.Billing.Feature.SitesAPI]
+        )
+
+        :ok
+      end
+
       test "can change domain name", %{conn: conn, site: site} do
         old_domain = site.domain
         assert old_domain != "new.example.com"
@@ -1878,8 +1985,8 @@ defmodule PlausibleWeb.Api.ExternalSitesControllerTest do
             "domain" => "new.example.com"
           })
 
-        res = json_response(conn, 404)
+        res = json_response(conn, 401)
-        assert res["error"] == "Site could not be found"
+        assert res["error"] =~ "Invalid API key"
       end
 
       test "can't make a no-op change", %{conn: conn, site: site} do