374 lines
13 KiB
Ruby
374 lines
13 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require 'spec_helper'
|
|
|
|
RSpec.describe Gitlab::Database::Partitioning, feature_category: :database do
|
|
include Database::PartitioningHelpers
|
|
include Database::TableSchemaHelpers
|
|
|
|
let(:main_connection) { ApplicationRecord.connection }
|
|
|
|
before do
|
|
stub_feature_flags(disallow_database_ddl_feature_flags: false)
|
|
end
|
|
|
|
around do |example|
|
|
previously_registered_models = described_class.registered_models.dup
|
|
described_class.instance_variable_set(:@registered_models, Set.new)
|
|
|
|
previously_registered_tables = described_class.registered_tables.dup
|
|
described_class.instance_variable_set(:@registered_tables, Set.new)
|
|
|
|
example.run
|
|
|
|
described_class.instance_variable_set(:@registered_models, previously_registered_models)
|
|
described_class.instance_variable_set(:@registered_tables, previously_registered_tables)
|
|
end
|
|
|
|
describe '.register_models' do
|
|
context 'ensure that the registered models have partitioning strategy' do
|
|
it 'fails when partitioning_strategy is not specified for the model' do
|
|
model = Class.new(ApplicationRecord)
|
|
expect { described_class.register_models([model]) }.to raise_error(/should have partitioning strategy defined/)
|
|
end
|
|
end
|
|
end
|
|
|
|
describe '.sync_partitions_ignore_db_error' do
|
|
it 'calls sync_partitions' do
|
|
expect(described_class).to receive(:sync_partitions).with(analyze: false)
|
|
|
|
described_class.sync_partitions_ignore_db_error
|
|
end
|
|
|
|
[ActiveRecord::ActiveRecordError, PG::Error].each do |error|
|
|
context "when #{error} is raised" do
|
|
before do
|
|
expect(described_class).to receive(:sync_partitions)
|
|
.and_raise(error)
|
|
end
|
|
|
|
it 'ignores it' do
|
|
described_class.sync_partitions_ignore_db_error
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'when DISABLE_POSTGRES_PARTITION_CREATION_ON_STARTUP is set' do
|
|
before do
|
|
stub_env('DISABLE_POSTGRES_PARTITION_CREATION_ON_STARTUP', '1')
|
|
end
|
|
|
|
it 'does not call sync_partitions' do
|
|
expect(described_class).not_to receive(:sync_partitions)
|
|
|
|
described_class.sync_partitions_ignore_db_error
|
|
end
|
|
end
|
|
end
|
|
|
|
describe '.sync_partitions' do
|
|
let(:ci_connection) { Ci::ApplicationRecord.connection }
|
|
let(:table_names) { %w[_test_partitioning_test1 _test_partitioning_test2] }
|
|
let(:models) do
|
|
[
|
|
Class.new(ApplicationRecord) do
|
|
include PartitionedTable
|
|
|
|
self.table_name = :_test_partitioning_test1
|
|
partitioned_by :created_at, strategy: :monthly
|
|
end,
|
|
Class.new(Gitlab::Database::Partitioning::TableWithoutModel).tap do |klass|
|
|
klass.table_name = :_test_partitioning_test2
|
|
klass.partitioned_by(:created_at, strategy: :monthly)
|
|
klass.limit_connection_names = %i[main]
|
|
end
|
|
]
|
|
end
|
|
|
|
before do
|
|
table_names.each do |table_name|
|
|
execute_on_each_database(<<~SQL)
|
|
CREATE TABLE #{table_name} (
|
|
id serial not null,
|
|
created_at timestamptz not null,
|
|
PRIMARY KEY (id, created_at))
|
|
PARTITION BY RANGE (created_at);
|
|
SQL
|
|
end
|
|
end
|
|
|
|
it 'manages partitions for each given model' do
|
|
expect { described_class.sync_partitions(models) }
|
|
.to change { find_partitions(table_names.first).size }.from(0)
|
|
.and change { find_partitions(table_names.last).size }.from(0)
|
|
end
|
|
|
|
context 'for analyze' do
|
|
let(:analyze_regex) { /ANALYZE / }
|
|
let(:analyze) { true }
|
|
|
|
shared_examples_for 'not running analyze' do
|
|
specify do
|
|
control = ActiveRecord::QueryRecorder.new { described_class.sync_partitions(analyze: analyze) }
|
|
expect(control.occurrences).not_to include(analyze_regex)
|
|
end
|
|
end
|
|
|
|
context 'when analyze_interval is not set' do
|
|
it_behaves_like 'not running analyze'
|
|
|
|
context 'when analyze is set to false' do
|
|
it_behaves_like 'not running analyze'
|
|
end
|
|
end
|
|
|
|
context 'when analyze_interval is set' do
|
|
let(:models) do
|
|
[
|
|
Class.new(ApplicationRecord) do
|
|
include PartitionedTable
|
|
|
|
self.table_name = :_test_partitioning_test1
|
|
partitioned_by :created_at, strategy: :monthly, analyze_interval: 1.week
|
|
end,
|
|
Class.new(Gitlab::Database::Partitioning::TableWithoutModel).tap do |klass|
|
|
klass.table_name = :_test_partitioning_test2
|
|
klass.partitioned_by(:created_at, strategy: :monthly, analyze_interval: 1.week)
|
|
klass.limit_connection_names = %i[main]
|
|
end
|
|
]
|
|
end
|
|
|
|
it 'runs analyze' do
|
|
control = ActiveRecord::QueryRecorder.new { described_class.sync_partitions(models, analyze: analyze) }
|
|
expect(control.occurrences).to include(analyze_regex)
|
|
end
|
|
|
|
context 'analyze is false' do
|
|
let(:analyze) { false }
|
|
|
|
it_behaves_like 'not running analyze'
|
|
end
|
|
end
|
|
end
|
|
|
|
context 'with multiple databases' do
|
|
it 'creates partitions in each database' do
|
|
skip_if_shared_database(:ci)
|
|
|
|
expect { described_class.sync_partitions(models) }
|
|
.to change { find_partitions(table_names.first, conn: main_connection).size }.from(0)
|
|
.and change { find_partitions(table_names.last, conn: main_connection).size }.from(0)
|
|
.and change { find_partitions(table_names.first, conn: ci_connection).size }.from(0)
|
|
.and change { find_partitions(table_names.last, conn: ci_connection).size }.from(0)
|
|
end
|
|
|
|
it 'does not create partitions in each database if restricted' do
|
|
skip_if_shared_database(:ci)
|
|
|
|
expect { described_class.sync_partitions(models, owner_db_only: true) }
|
|
.to change { find_partitions(table_names.first, conn: main_connection).size }.from(0)
|
|
.and change { find_partitions(table_names.last, conn: main_connection).size }.from(0)
|
|
.and change { find_partitions(table_names.first, conn: ci_connection).size }.by_at_most(0)
|
|
.and change { find_partitions(table_names.last, conn: ci_connection).size }.by_at_most(0)
|
|
end
|
|
end
|
|
|
|
context 'without ci database' do
|
|
it 'only creates partitions for main database' do
|
|
skip_if_database_exists(:ci)
|
|
|
|
allow(Gitlab::Database::Partitioning::PartitionManager).to receive(:new).and_call_original
|
|
|
|
# Also, in the case where `ci` database is shared with `main` database,
|
|
# check that we do not run PartitionManager again for ci connection as
|
|
# that is redundant.
|
|
expect(Gitlab::Database::Partitioning::PartitionManager).not_to receive(:new)
|
|
.with(anything, connection: ci_connection).and_call_original
|
|
|
|
expect { described_class.sync_partitions(models) }
|
|
.to change { find_partitions(table_names.first, conn: main_connection).size }.from(0)
|
|
.and change { find_partitions(table_names.last, conn: main_connection).size }.from(0)
|
|
end
|
|
end
|
|
|
|
context 'when no partitioned models are given' do
|
|
it 'manages partitions for each registered model' do
|
|
described_class.register_models([models.first])
|
|
described_class.register_tables(
|
|
[
|
|
{
|
|
table_name: table_names.last,
|
|
partitioned_column: :created_at,
|
|
strategy: :monthly
|
|
}
|
|
])
|
|
|
|
expect { described_class.sync_partitions }
|
|
.to change { find_partitions(table_names.first).size }.from(0)
|
|
.and change { find_partitions(table_names.last).size }.from(0)
|
|
end
|
|
end
|
|
|
|
context 'when only a specific database is requested' do
|
|
let(:ci_model) do
|
|
Class.new(Ci::ApplicationRecord) do
|
|
include PartitionedTable
|
|
|
|
self.table_name = :_test_partitioning_test3
|
|
partitioned_by :created_at, strategy: :monthly
|
|
end
|
|
end
|
|
|
|
before do
|
|
skip_if_shared_database(:ci)
|
|
|
|
(table_names + [:_test_partitioning_test3]).each do |table_name|
|
|
execute_on_each_database("DROP TABLE IF EXISTS #{table_name}")
|
|
|
|
execute_on_each_database(<<~SQL)
|
|
CREATE TABLE #{table_name} (
|
|
id serial not null,
|
|
created_at timestamptz not null,
|
|
PRIMARY KEY (id, created_at))
|
|
PARTITION BY RANGE (created_at);
|
|
SQL
|
|
end
|
|
end
|
|
|
|
after do
|
|
(table_names + [:_test_partitioning_test3]).each do |table_name|
|
|
ci_connection.execute("DROP TABLE IF EXISTS #{table_name}")
|
|
end
|
|
end
|
|
|
|
it 'manages partitions for models for the given database', :aggregate_failures do
|
|
expect { described_class.sync_partitions([models.first, ci_model], only_on: 'ci') }
|
|
.to change { find_partitions(ci_model.table_name, conn: ci_connection).size }.from(0)
|
|
|
|
expect(find_partitions(models.first.table_name, conn: main_connection).size).to eq(0)
|
|
expect(find_partitions(models.first.table_name, conn: ci_connection).size).to eq(0)
|
|
expect(find_partitions(ci_model.table_name, conn: main_connection).size).to eq(0)
|
|
end
|
|
end
|
|
|
|
context 'when partition_manager_sync_partitions feature flag is disabled' do
|
|
before do
|
|
described_class.register_models(models)
|
|
stub_feature_flags(partition_manager_sync_partitions: false)
|
|
end
|
|
|
|
it 'skips sync_partitions' do
|
|
expect(described_class::PartitionManager).not_to receive(:new)
|
|
expect(described_class).to receive(:sync_partitions)
|
|
.and_call_original
|
|
|
|
described_class.sync_partitions(models)
|
|
end
|
|
end
|
|
|
|
context 'when disallow_database_ddl_feature_flags feature flag is enabled' do
|
|
before do
|
|
described_class.register_models(models)
|
|
stub_feature_flags(disallow_database_ddl_feature_flags: true)
|
|
end
|
|
|
|
it 'skips sync_partitions' do
|
|
expect(described_class::PartitionManager).not_to receive(:new)
|
|
expect(described_class).to receive(:sync_partitions).and_call_original
|
|
|
|
described_class.sync_partitions(models)
|
|
end
|
|
end
|
|
end
|
|
|
|
describe '.report_metrics' do
|
|
let(:model1) { double('model') }
|
|
let(:model2) { double('model') }
|
|
|
|
let(:partition_monitoring_class) { described_class::PartitionMonitoring }
|
|
|
|
context 'when no partitioned models are given' do
|
|
it 'reports metrics for each registered model' do
|
|
expect_next_instance_of(partition_monitoring_class) do |partition_monitor|
|
|
expect(partition_monitor).to receive(:report_metrics_for_model).with(model1)
|
|
expect(partition_monitor).to receive(:report_metrics_for_model).with(model2)
|
|
end
|
|
|
|
expect(Gitlab::Database::EachDatabase).to receive(:each_model_connection)
|
|
.with(described_class.__send__(:registered_models))
|
|
.and_yield(model1)
|
|
.and_yield(model2)
|
|
|
|
described_class.report_metrics
|
|
end
|
|
end
|
|
|
|
context 'when partitioned models are given' do
|
|
it 'reports metrics for each given model' do
|
|
expect_next_instance_of(partition_monitoring_class) do |partition_monitor|
|
|
expect(partition_monitor).to receive(:report_metrics_for_model).with(model1)
|
|
expect(partition_monitor).to receive(:report_metrics_for_model).with(model2)
|
|
end
|
|
|
|
expect(Gitlab::Database::EachDatabase).to receive(:each_model_connection)
|
|
.with([model1, model2])
|
|
.and_yield(model1)
|
|
.and_yield(model2)
|
|
|
|
described_class.report_metrics([model1, model2])
|
|
end
|
|
end
|
|
end
|
|
|
|
describe '.drop_detached_partitions' do
|
|
let(:table_names) { %w[_test_detached_test_partition1 _test_detached_test_partition2] }
|
|
|
|
before do
|
|
table_names.each do |table_name|
|
|
connection.create_table("#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{table_name}")
|
|
|
|
Postgresql::DetachedPartition.create!(table_name: table_name, drop_after: 1.year.ago)
|
|
end
|
|
end
|
|
|
|
it 'drops detached partitions for each database' do
|
|
expect(Gitlab::Database::EachDatabase).to receive(:each_connection).and_yield
|
|
|
|
expect { described_class.drop_detached_partitions }
|
|
.to change { Postgresql::DetachedPartition.count }.from(2).to(0)
|
|
.and change { table_exists?(table_names.first) }.from(true).to(false)
|
|
.and change { table_exists?(table_names.last) }.from(true).to(false)
|
|
end
|
|
|
|
context 'when the feature flag is disabled' do
|
|
before do
|
|
stub_feature_flags(partition_manager_sync_partitions: false)
|
|
end
|
|
|
|
it 'does not call the DetachedPartitionDropper' do
|
|
expect(Gitlab::Database::Partitioning::DetachedPartitionDropper).not_to receive(:new)
|
|
|
|
described_class.drop_detached_partitions
|
|
end
|
|
end
|
|
|
|
context 'when the feature disallow DDL feature flags is enabled' do
|
|
before do
|
|
stub_feature_flags(disallow_database_ddl_feature_flags: true)
|
|
end
|
|
|
|
it 'does not call the DetachedPartitionDropper' do
|
|
expect(Gitlab::Database::Partitioning::DetachedPartitionDropper).not_to receive(:new)
|
|
|
|
described_class.drop_detached_partitions
|
|
end
|
|
end
|
|
|
|
def table_exists?(table_name)
|
|
table_oid(table_name).present?
|
|
end
|
|
end
|
|
end
|