diff --git a/config/runtime.exs b/config/runtime.exs index 7d6f5d4ce0..5069ce500e 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -560,6 +560,9 @@ config :plausible, Plausible.IngestRepo, pool_size: ingest_pool_size, settings: [ materialized_views_ignore_errors: 1 + ], + table_settings: [ + storage_policy: get_var_from_path_or_env(config_dir, "CLICKHOUSE_DEFAULT_STORAGE_POLICY") ] config :plausible, Plausible.AsyncInsertRepo, diff --git a/lib/plausible/data_migration/acquisition_channel.ex b/lib/plausible/data_migration/acquisition_channel.ex index 34dbe2c5e2..754a8dbe54 100644 --- a/lib/plausible/data_migration/acquisition_channel.ex +++ b/lib/plausible/data_migration/acquisition_channel.ex @@ -25,6 +25,7 @@ defmodule Plausible.DataMigration.AcquisitionChannel do "acquisition_channel_functions", [ on_cluster_statement: on_cluster_statement, + table_settings: Plausible.MigrationUtils.table_settings_expr(), dictionary_connection_params: Plausible.MigrationUtils.dictionary_connection_params(), insert_quorum: insert_quorum ], diff --git a/lib/plausible/data_migration/locations_sync.ex b/lib/plausible/data_migration/locations_sync.ex index 7ebba6098b..b2708e16ff 100644 --- a/lib/plausible/data_migration/locations_sync.ex +++ b/lib/plausible/data_migration/locations_sync.ex @@ -91,7 +91,12 @@ defmodule Plausible.DataMigration.LocationsSync do cluster? = Plausible.IngestRepo.clustered_table?("sessions_v2") {:ok, _} = run_sql("truncate-location-data-table", cluster?: cluster?) - {:ok, _} = run_sql("create-location-data-table", cluster?: cluster?) + + {:ok, _} = + run_sql("create-location-data-table", + cluster?: cluster?, + table_settings: Plausible.MigrationUtils.table_settings_expr(:suffix) + ) countries = Location.Country.all() diff --git a/lib/plausible/data_migration/numeric_ids.ex b/lib/plausible/data_migration/numeric_ids.ex index 1980709ad8..43092d2694 100644 --- a/lib/plausible/data_migration/numeric_ids.ex +++ b/lib/plausible/data_migration/numeric_ids.ex @@ -18,8 +18,6 @@ defmodule Plausible.DataMigration.NumericIDs do end end - @table_settings "SETTINGS index_granularity = 8192" - def run(opts \\ []) do interactive? = Keyword.get(opts, :interactive?, true) @@ -34,7 +32,7 @@ defmodule Plausible.DataMigration.NumericIDs do table_settings = Keyword.get(opts, :table_settings) || System.get_env("NUMERIC_IDS_TABLE_SETTINGS") || - @table_settings + Plausible.MigrationUtils.table_settings_expr() start_from = Keyword.get(opts, :start_from) || System.get_env("NUMERIC_IDS_PARTITION_START_FROM") diff --git a/lib/plausible/migration_utils.ex b/lib/plausible/migration_utils.ex index e477537e76..93cbca632e 100644 --- a/lib/plausible/migration_utils.ex +++ b/lib/plausible/migration_utils.ex @@ -21,4 +21,23 @@ defmodule Plausible.MigrationUtils do |> Enum.reject(&is_nil/1) |> Enum.join(" ") end + + def table_settings() do + IngestRepo.config() + |> Keyword.get(:table_settings) + |> Enum.reject(fn {_, v} -> is_nil(v) end) + end + + def table_settings_expr(type \\ :prefix) do + expr = Enum.map_join(table_settings(), ", ", fn {k, v} -> "#{k} = #{encode(v)}" end) + + case {table_settings(), type} do + {[], _} -> "" + {_, :prefix} -> "SETTINGS #{expr}" + {_, :suffix} -> ", #{expr}" + end + end + + defp encode(value) when is_number(value), do: value + defp encode(value) when is_binary(value), do: "'#{value}'" end diff --git a/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex index 742edca9fe..7287f7f57f 100644 --- a/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex +++ b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex @@ -9,7 +9,8 @@ ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_pro <% else %> ENGINE = MergeTree() <% end %> -ORDER BY referrer_source; +ORDER BY referrer_source +<%= @table_settings %>; TRUNCATE TABLE acquisition_channel_source_category SETTINGS alter_sync=2; @@ -41,7 +42,8 @@ ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_pro <% else %> ENGINE = MergeTree() <% end %> -ORDER BY referrer_source; +ORDER BY referrer_source +<%= @table_settings %>; TRUNCATE TABLE acquisition_channel_paid_sources SETTINGS alter_sync=2; diff --git a/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex b/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex index 0487c5c7e9..f5e5593f75 100644 --- a/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex +++ b/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex @@ -10,4 +10,4 @@ ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_pro ENGINE = MergeTree() <% end %> ORDER BY (type, id) -SETTINGS index_granularity = 128 +SETTINGS index_granularity = 128 <%= @table_settings %> diff --git a/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs index 6f329cdfb9..527612827a 100644 --- a/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs +++ b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs @@ -10,8 +10,11 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions do create_if_not_exists table(:events, primary_key: false, engine: "MergeTree", - options: - "PARTITION BY toYYYYMM(timestamp) ORDER BY (domain, toDate(timestamp), user_id) SETTINGS index_granularity = 8192" + options: """ + PARTITION BY toYYYYMM(timestamp) + ORDER BY (domain, toDate(timestamp), user_id) + #{Plausible.MigrationUtils.table_settings_expr()} + """ ) do add(:name, :string) add(:domain, :string) @@ -34,8 +37,11 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions do create_if_not_exists table(:sessions, primary_key: false, engine: "CollapsingMergeTree(sign)", - options: - "PARTITION BY toYYYYMM(start) ORDER BY (domain, toDate(start), user_id, session_id) SETTINGS index_granularity = 8192" + options: """ + PARTITION BY toYYYYMM(start) + ORDER BY (domain, toDate(start), user_id, session_id) + #{Plausible.MigrationUtils.table_settings_expr()} + """ ) do add(:session_id, :UInt64) add(:sign, :Int8) diff --git a/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs index 5c7b72a875..cd1d827db5 100644 --- a/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs +++ b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs @@ -5,7 +5,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do create_if_not_exists table(:imported_visitors, primary_key: false, engine: "MergeTree", - options: "ORDER BY (site_id, date)" + options: """ + ORDER BY (site_id, date) + #{Plausible.MigrationUtils.table_settings_expr()} + """ ) do add(:site_id, :UInt64) add(:date, :date) @@ -19,7 +22,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do create_if_not_exists table(:imported_sources, primary_key: false, engine: "MergeTree", - options: "ORDER BY (site_id, date, source)" + options: """ + ORDER BY (site_id, date, source) + #{Plausible.MigrationUtils.table_settings_expr()} + """ ) do add(:site_id, :UInt64) add(:date, :date) @@ -37,7 +43,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do create_if_not_exists table(:imported_pages, primary_key: false, engine: "MergeTree", - options: "ORDER BY (site_id, date, hostname, page)" + options: """ + ORDER BY (site_id, date, hostname, page) + #{Plausible.MigrationUtils.table_settings_expr()} + """ ) do add(:site_id, :UInt64) add(:date, :date) @@ -52,7 +61,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do create_if_not_exists table(:imported_entry_pages, primary_key: false, engine: "MergeTree", - options: "ORDER BY (site_id, date, entry_page)" + options: """ + ORDER BY (site_id, date, entry_page) + #{Plausible.MigrationUtils.table_settings_expr()} + """ ) do add(:site_id, :UInt64) add(:date, :date) @@ -66,7 +78,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do create_if_not_exists table(:imported_exit_pages, primary_key: false, engine: "MergeTree", - options: "ORDER BY (site_id, date, exit_page)" + options: """ + ORDER BY (site_id, date, exit_page) + #{Plausible.MigrationUtils.table_settings_expr()} + """ ) do add(:site_id, :UInt64) add(:date, :date) @@ -78,7 +93,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do create_if_not_exists table(:imported_locations, primary_key: false, engine: "MergeTree", - options: "ORDER BY (site_id, date, country, region, city)" + options: """ + ORDER BY (site_id, date, country, region, city) + #{Plausible.MigrationUtils.table_settings_expr()} + """ ) do add(:site_id, :UInt64) add(:date, :date) @@ -94,7 +112,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do create_if_not_exists table(:imported_devices, primary_key: false, engine: "MergeTree", - options: "ORDER BY (site_id, date, device)" + options: """ + ORDER BY (site_id, date, device) + #{Plausible.MigrationUtils.table_settings_expr()} + """ ) do add(:site_id, :UInt64) add(:date, :date) @@ -108,7 +129,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do create_if_not_exists table(:imported_browsers, primary_key: false, engine: "MergeTree", - options: "ORDER BY (site_id, date, browser)" + options: """ + ORDER BY (site_id, date, browser) + #{Plausible.MigrationUtils.table_settings_expr()} + """ ) do add(:site_id, :UInt64) add(:date, :date) @@ -122,7 +146,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do create_if_not_exists table(:imported_operating_systems, primary_key: false, engine: "MergeTree", - options: "ORDER BY (site_id, date, operating_system)" + options: """ + ORDER BY (site_id, date, operating_system) + #{Plausible.MigrationUtils.table_settings_expr()} + """ ) do add(:site_id, :UInt64) add(:date, :date) diff --git a/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs b/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs index 75596571c0..d1e2b4a9f1 100644 --- a/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs +++ b/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs @@ -5,8 +5,10 @@ defmodule Plausible.IngestRepo.Migrations.CreateIngestCountersTable do create_if_not_exists table(:ingest_counters, primary_key: false, engine: "SummingMergeTree(value)", - options: - "ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))" + options: """ + ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket)) + #{Plausible.MigrationUtils.table_settings_expr()} + """ ) do add(:event_timebucket, :utc_datetime) add(:domain, :"LowCardinality(String)") diff --git a/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs index b881f5eae6..00de3b0c3d 100644 --- a/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs +++ b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs @@ -1,10 +1,10 @@ defmodule Plausible.IngestRepo.Migrations.CreateV2Schemas do @moduledoc """ Normally, for live environments the migration will be done via - `DataMigration.NumericIDs` module. + `DataMigration.NumericIDs` module. - For tests, and entirely new small, self-hosted instances however, - we want to keep the ability of preparing the database without enforcing + For tests, and entirely new small, self-hosted instances however, + we want to keep the ability of preparing the database without enforcing any _data_ migration. """ @@ -13,11 +13,12 @@ defmodule Plausible.IngestRepo.Migrations.CreateV2Schemas do use Plausible.DataMigration, dir: "NumericIDs" @cluster? false - @settings "SETTINGS index_granularity = 8192" def up do - execute unwrap("create-events-v2", table_settings: @settings, cluster?: @cluster?) - execute unwrap("create-sessions-v2", table_settings: @settings, cluster?: @cluster?) + table_settings = Plausible.MigrationUtils.table_settings_expr() + + execute unwrap("create-events-v2", table_settings: table_settings, cluster?: @cluster?) + execute unwrap("create-sessions-v2", table_settings: table_settings, cluster?: @cluster?) end def down do diff --git a/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs b/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs index 0cc2f76b7f..549f86de4f 100644 --- a/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs +++ b/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs @@ -4,19 +4,20 @@ defmodule Plausible.IngestRepo.Migrations.AddImportedCustomEvents do def change do # NOTE: Using another table for determining cluster presence on_cluster = Plausible.MigrationUtils.on_cluster_statement("imported_pages") + table_settings = Plausible.MigrationUtils.table_settings_expr(:suffix) settings = if Plausible.IngestRepo.clustered_table?("imported_pages") do """ ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/{database}/imported_custom_events', '{replica}') ORDER BY (site_id, import_id, date, name) - SETTINGS replicated_deduplication_window = 0, storage_policy = 'tiered' + SETTINGS replicated_deduplication_window = 0 #{table_settings} """ else """ ENGINE = MergeTree() ORDER BY (site_id, import_id, date, name) - SETTINGS replicated_deduplication_window = 0 + SETTINGS replicated_deduplication_window = 0 #{table_settings} """ end