root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2021-07-06 18:08:17 +00:00
parent b72b14cb4d
commit 971f05815d
26 changed files with 442 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
GITALY_SERVER_VERSION
View File

@ -1 +1 @@
b84ba4f096da54ebb6a85c14ab736474c72f1a2a
d12fb69a841d91d843f392a124865f6d47d3bc22

2
app/controllers/admin/application_settings_controller.rb
View File

@ -27,7 +27,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
feature_category :source_code_management, [:repository, :clear_repository_check_states]
feature_category :continuous_integration, [:ci_cd, :reset_registration_token]
feature_category :usage_ping, [:usage_data]
feature_category :service_ping, [:usage_data]
feature_category :integrations, [:integrations]
feature_category :pages, [:lets_encrypt_terms_of_service]

2
app/controllers/projects/service_ping_controller.rb
View File

@ -3,7 +3,7 @@
class Projects::ServicePingController < Projects::ApplicationController
before_action :authenticate_user!
feature_category :usage_ping
feature_category :service_ping
def web_ide_clientside_preview
return render_404 unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled?

9
app/helpers/issuables_helper.rb
View File

@ -425,15 +425,6 @@ module IssuablesHelper
}
end
def sidebar_status_data(issuable_sidebar, project)
{
iid: issuable_sidebar[:iid],
issuable_type: issuable_sidebar[:type],
full_path: project.full_path,
can_edit: issuable_sidebar.dig(:current_user, :can_edit).to_s
}
end
def parent
@project || @group
end

1
app/models/integrations/jenkins.rb
View File

@ -97,7 +97,6 @@ module Integrations
{
type: 'text',
name: 'username',
required: true,
help: s_('The username for the Jenkins server.')
},
{

2
app/views/shared/issuable/_sidebar.html.haml
View File

@ -81,7 +81,7 @@
#js-severity
- if issuable_sidebar.dig(:features_available, :health_status)
.js-sidebar-status-entry-point{ data: sidebar_status_data(issuable_sidebar, @project) }
.js-sidebar-status-entry-point
- if issuable_sidebar.has_key?(:confidential)
%script#js-confidential-issue-data{ type: "application/json" }= { is_confidential: issuable_sidebar[:confidential], is_editable: can_edit_issuable }.to_json.html_safe

2
app/workers/all_queues.yml
View File

@ -267,7 +267,7 @@
:tags: []
- :name: cronjob:gitlab_usage_ping
:worker_name: GitlabUsagePingWorker
:feature_category: :usage_ping
:feature_category: :service_ping
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown

2
app/workers/gitlab_usage_ping_worker.rb
View File

@ -8,7 +8,7 @@ class GitlabUsagePingWorker # rubocop:disable Scalability/IdempotentWorker
include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
include Gitlab::ExclusiveLeaseHelpers
feature_category :usage_ping
feature_category :service_ping
sidekiq_options retry: 3, dead: false
sidekiq_retry_in { |count| (count + 1) * 8.hours.to_i }

8
config/feature_flags/development/database_reindexing_pg12.yml Normal file
View File

@ -0,0 +1,8 @@
---
name: database_reindexing_pg12
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64695
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/334372
milestone: '14.1'
type: development
group: group::database
default_enabled: false

14
doc/development/internal_api.md
View File

@ -529,7 +529,7 @@ POST /namespaces/:id/gitlab_subscription
Example request:
```shell
curl --request POST --header "TOKEN: <admin_access_token>" "https://gitlab.com/api/v4/namespaces/1234/gitlab_subscription?start_date="2020-07-15"&plan="silver"&seats=10"
curl --request POST --header "TOKEN: <admin_access_token>" "https://gitlab.com/api/v4/namespaces/1234/gitlab_subscription?start_date="2020-07-15"&plan="premium"&seats=10"
```
Example response:
@ -537,8 +537,8 @@ Example response:
```json
{
"plan": {
"code":"silver",
"name":"Silver",
"code":"premium",
"name":"premium",
"trial":false,
"auto_renew":null,
"upgradable":false
@ -588,8 +588,8 @@ Example response:
```json
{
"plan": {
"code":"silver",
"name":"Silver",
"code":"premium",
"name":"premium",
"trial":false,
"auto_renew":null,
"upgradable":false
@ -627,8 +627,8 @@ Example response:
```json
{
"plan": {
"code":"silver",
"name":"Silver",
"code":"premium",
"name":"premium",
"trial":false,
"auto_renew":null,
"upgradable":false

13
doc/user/clusters/agent/ci_cd_tunnel.md
View File

@ -30,30 +30,29 @@ To create the Tunnel:
.kubectl_config: &kubectl_config
- |
cat << EOF > "$HOME/agent_config.yaml"
cat << EOF > "$CI_PROJECT_DIR/.kubeconfig.agent.yaml"
apiVersion: v1
kind: Config
clusters:
- cluster:
server: https://kas.gitlab.com/k8s-proxy
server: https://kas.gitlab.com/k8s-proxy
name: agent
users:
- name: agent
user:
token: "ci:$AGENT_ID:$CI_JOB_TOKEN"
token: "ci:$AGENT_ID:$CI_JOB_TOKEN"
contexts:
- context:
cluster: agent
user: agent
name: agent
user: agent
name: agent
current-context: agent
EOF
- export KUBECONFIG="$KUBECONFIG:$HOME/agent_config.yaml"
deploy:
script:
- *kubectl_config
- kubectl get pods
- kubectl --kubeconfig="$CI_PROJECT_DIR/.kubeconfig.agent.yaml" get pods
```
1. Execute `kubectl` commands directly against your cluster with this CI/CD job you just created.

4
doc/user/project/repository/branches/default.md
View File

@ -57,9 +57,7 @@ GitLab administrators can configure a new default branch name at the
### Instance-level custom initial branch name **(FREE SELF)**
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221013) in GitLab 13.2.
> - It's deployed behind a feature flag, enabled by default.
> - It cannot be enabled or disabled per-project.
> - It's recommended for production use.
> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/325163) in GitLab 13.12.
GitLab [administrators](../../../permissions.md) of self-managed instances can
customize the initial branch for projects hosted on that instance. Individual

5
doc/user/project/repository/repository_mirroring.md
View File

@ -320,8 +320,9 @@ For more information, see [Start the pull mirroring process for a Project](../..
> - Moved to GitLab Premium in 13.9.
Based on the mirror direction that you choose, you can opt to mirror only the
[protected branches](../protected_branches.md) from/to your remote repository.
For pull mirroring, non-protected branches are not mirrored and can diverge.
[protected branches](../protected_branches.md) in the mirroring project,
either from or to your remote repository. For pull mirroring, non-protected branches in
the mirroring project are not mirrored and can diverge.
To use this option, check the **Only mirror protected branches** box when
creating a repository mirror. **(PREMIUM)**

2
lib/api/usage_data.rb
View File

@ -4,7 +4,7 @@ module API
class UsageData < ::API::Base
before { authenticate_non_get! }
feature_category :usage_ping
feature_category :service_ping
namespace 'usage_data' do
before do

2
lib/api/usage_data_non_sql_metrics.rb
View File

@ -4,7 +4,7 @@ module API
class UsageDataNonSqlMetrics < ::API::Base
before { authenticated_as_admin! }
feature_category :usage_ping
feature_category :service_ping
namespace 'usage_data' do
before do

2
lib/api/usage_data_queries.rb
View File

@ -4,7 +4,7 @@ module API
class UsageDataQueries < ::API::Base
before { authenticated_as_admin! }
feature_category :usage_ping
feature_category :service_ping
namespace 'usage_data' do
before do

11
lib/gitlab/database/postgres_index.rb
View File

@ -20,16 +20,25 @@ module Gitlab
# A 'regular' index is a non-unique index,
# that does not serve an exclusion constraint and
# is defined on a table that is not partitioned.
scope :regular, -> { where(unique: false, partitioned: false, exclusion: false)}
#
# Deprecated: Switch to scope .reindexing_support
scope :regular, -> { where(unique: false, partitioned: false, exclusion: false, expression: false)}
# Indexes for reindexing with PG12
scope :reindexing_support, -> { where(partitioned: false, exclusion: false, expression: false) }
scope :not_match, ->(regex) { where("name !~ ?", regex)}
scope :match, ->(regex) { where("name ~* ?", regex)}
scope :not_recently_reindexed, -> do
recent_actions = Reindexing::ReindexAction.recent.where('index_identifier = identifier')
where('NOT EXISTS (?)', recent_actions)
end
alias_method :reset, :reload
def bloat_size
strong_memoize(:bloat_size) { bloat_estimate&.bloat_size || 0 }
end

11
lib/gitlab/database/reindexing.rb
View File

@ -14,11 +14,16 @@ module Gitlab
end
def self.candidate_indexes
Gitlab::Database::PostgresIndex
.regular
.where('NOT expression')
indexes = Gitlab::Database::PostgresIndex
.not_match("^#{ConcurrentReindex::TEMPORARY_INDEX_PREFIX}")
.not_match("^#{ConcurrentReindex::REPLACED_INDEX_PREFIX}")
.not_match("#{ReindexConcurrently::TEMPORARY_INDEX_PATTERN}$")
if Feature.enabled?(:database_reindexing_pg12, type: :development)
indexes.reindexing_support
else
indexes.regular
end
end
end
end

8
lib/gitlab/database/reindexing/coordinator.rb
View File

@ -41,7 +41,13 @@ module Gitlab
end
def perform_for(index, action)
ConcurrentReindex.new(index).perform
strategy = if Feature.enabled?(:database_reindexing_pg12, type: :development)
ReindexConcurrently
else
ConcurrentReindex
end
strategy.new(index).perform
rescue StandardError
action.state = :failed

136
lib/gitlab/database/reindexing/reindex_concurrently.rb Normal file
View File

@ -0,0 +1,136 @@
# frozen_string_literal: true
module Gitlab
module Database
module Reindexing
# This is a >= PG12 reindexing strategy based on `REINDEX CONCURRENTLY`
class ReindexConcurrently
ReindexError = Class.new(StandardError)
TEMPORARY_INDEX_PATTERN = '\_ccnew[0-9]*'
STATEMENT_TIMEOUT = 9.hours
PG_MAX_INDEX_NAME_LENGTH = 63
# When dropping an index, we acquire a SHARE UPDATE EXCLUSIVE lock,
# which only conflicts with DDL and vacuum. We therefore execute this with a rather
# high lock timeout and a long pause in between retries. This is an alternative to
# setting a high statement timeout, which would lead to a long running query with effects
# on e.g. vacuum.
REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30
attr_reader :index, :logger
def initialize(index, logger: Gitlab::AppLogger)
@index = index
@logger = logger
end
def perform
raise ReindexError, 'indexes serving an exclusion constraint are currently not supported' if index.exclusion?
raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if index.name =~ /#{TEMPORARY_INDEX_PATTERN}/
# Expression indexes require additional statistics in `pg_statistic`:
# select * from pg_statistic where starelid = (select oid from pg_class where relname = 'some_index');
#
# In PG12, this has been fixed in https://gitlab.com/postgres/postgres/-/commit/b17ff07aa3eb142d2cde2ea00e4a4e8f63686f96.
# Discussion happened in https://www.postgresql.org/message-id/flat/CAFcNs%2BqpFPmiHd1oTXvcPdvAHicJDA9qBUSujgAhUMJyUMb%2BSA%40mail.gmail.com
# following a GitLab.com incident that surfaced this (https://gitlab.com/gitlab-com/gl-infra/production/-/issues/2885).
#
# While this has been backpatched, we continue to disable expression indexes until further review.
raise ReindexError, 'expression indexes are currently not supported' if index.expression?
begin
with_logging do
set_statement_timeout do
execute("REINDEX INDEX CONCURRENTLY #{quote_table_name(index.schema)}.#{quote_table_name(index.name)}")
end
end
ensure
cleanup_dangling_indexes
end
end
private
def with_logging
bloat_size = index.bloat_size
ondisk_size_before = index.ondisk_size_bytes
logger.info(
message: "Starting reindex of #{index}",
index: index.identifier,
table: index.tablename,
estimated_bloat_bytes: bloat_size,
index_size_before_bytes: ondisk_size_before
)
duration = Benchmark.realtime do
yield
end
index.reset
logger.info(
message: "Finished reindex of #{index}",
index: index.identifier,
table: index.tablename,
estimated_bloat_bytes: bloat_size,
index_size_before_bytes: ondisk_size_before,
index_size_after_bytes: index.ondisk_size_bytes,
duration_s: duration.round(2)
)
end
def cleanup_dangling_indexes
Gitlab::Database::PostgresIndex.match("#{TEMPORARY_INDEX_PATTERN}$").each do |lingering_index|
# Example lingering index name: some_index_ccnew1
# Example prefix: 'some_index'
prefix = lingering_index.name.gsub(/#{TEMPORARY_INDEX_PATTERN}/, '')
# Example suffix: '_ccnew1'
suffix = lingering_index.name.match(/#{TEMPORARY_INDEX_PATTERN}/)[0]
# Only remove if the lingering index name could have been chosen
# as a result of a REINDEX operation (considering that PostgreSQL
# truncates index names to 63 chars and adds a suffix).
if index.name[0...PG_MAX_INDEX_NAME_LENGTH - suffix.length] == prefix
remove_index(lingering_index)
end
end
end
def remove_index(index)
logger.info("Removing dangling index #{index.identifier}")
retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new(
timing_configuration: REMOVE_INDEX_RETRY_CONFIG,
klass: self.class,
logger: logger
)
retries.run(raise_on_exhaustion: false) do
execute("DROP INDEX CONCURRENTLY IF EXISTS #{quote_table_name(index.schema)}.#{quote_table_name(index.name)}")
end
end
def with_lock_retries(&block)
arguments = { klass: self.class, logger: logger }
Gitlab::Database::WithLockRetries.new(**arguments).run(raise_on_exhaustion: true, &block)
end
def set_statement_timeout
execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT)
yield
ensure
execute('RESET statement_timeout')
end
delegate :execute, :quote_table_name, to: :connection
def connection
@connection ||= ActiveRecord::Base.connection
end
end
end
end
end

6
locale/gitlab.pot
View File

@ -12832,9 +12832,6 @@ msgstr ""
msgid "Error occurred when saving reviewers"
msgstr ""
msgid "Error occurred while updating the %{issuableType} status"
msgstr ""
msgid "Error occurred while updating the issue status"
msgstr ""
@ -30474,9 +30471,6 @@ msgstr ""
msgid "Something went wrong while setting %{issuableType} confidentiality."
msgstr ""
msgid "Something went wrong while setting %{issuableType} health status."
msgstr ""
msgid "Something went wrong while setting %{issuableType} notifications."
msgstr ""

12
spec/db/schema_spec.rb
View File

@ -264,6 +264,18 @@ RSpec.describe 'Database schema' do
end
end
context 'index names' do
it 'disallows index names with a _ccnew[0-9]* suffix' do
# During REINDEX operations, Postgres generates a temporary index with a _ccnew[0-9]* suffix
# Since indexes are being considered temporary and subject to removal if they stick around for longer. See Gitlab::Database::Reindexing.
#
# Hence we disallow adding permanent indexes with this suffix.
problematic_indexes = Gitlab::Database::PostgresIndex.match("#{Gitlab::Database::Reindexing::ReindexConcurrently::TEMPORARY_INDEX_PATTERN}$").all
expect(problematic_indexes).to be_empty
end
end
private
def retrieve_columns_name_with_jsonb

18
spec/lib/gitlab/database/postgres_index_spec.rb
View File

@ -34,6 +34,24 @@ RSpec.describe Gitlab::Database::PostgresIndex do
it 'only indexes that dont serve an exclusion constraint' do
expect(described_class.regular).to all(have_attributes(exclusion: false))
end
it 'only non-expression indexes' do
expect(described_class.regular).to all(have_attributes(expression: false))
end
end
describe '.reindexing_support' do
it 'only non partitioned indexes' do
expect(described_class.reindexing_support).to all(have_attributes(partitioned: false))
end
it 'only indexes that dont serve an exclusion constraint' do
expect(described_class.reindexing_support).to all(have_attributes(exclusion: false))
end
it 'only non-expression indexes' do
expect(described_class.reindexing_support).to all(have_attributes(expression: false))
end
end
describe '.not_match' do

101
spec/lib/gitlab/database/reindexing/coordinator_spec.rb
View File

@ -9,13 +9,6 @@ RSpec.describe Gitlab::Database::Reindexing::Coordinator do
describe '.perform' do
subject { described_class.new(index, notifier).perform }
before do
swapout_view_for_table(:postgres_indexes)
allow(Gitlab::Database::Reindexing::ConcurrentReindex).to receive(:new).with(index).and_return(reindexer)
allow(Gitlab::Database::Reindexing::ReindexAction).to receive(:create_for).with(index).and_return(action)
end
let(:index) { create(:postgres_index) }
let(:notifier) { instance_double(Gitlab::Database::Reindexing::GrafanaNotifier, notify_start: nil, notify_end: nil) }
let(:reindexer) { instance_double(Gitlab::Database::Reindexing::ConcurrentReindex, perform: nil) }
@ -26,57 +19,83 @@ RSpec.describe Gitlab::Database::Reindexing::Coordinator do
let(:lease_timeout) { 1.day }
let(:uuid) { 'uuid' }
context 'locking' do
it 'acquires a lock while reindexing' do
expect(lease).to receive(:try_obtain).ordered.and_return(uuid)
shared_examples_for 'reindexing coordination' do
context 'locking' do
it 'acquires a lock while reindexing' do
expect(lease).to receive(:try_obtain).ordered.and_return(uuid)
expect(reindexer).to receive(:perform).ordered
expect(reindexer).to receive(:perform).ordered
expect(Gitlab::ExclusiveLease).to receive(:cancel).ordered.with(lease_key, uuid)
expect(Gitlab::ExclusiveLease).to receive(:cancel).ordered.with(lease_key, uuid)
subject
subject
end
it 'does not perform reindexing actions if lease is not granted' do
expect(lease).to receive(:try_obtain).ordered.and_return(false)
expect(Gitlab::Database::Reindexing::ConcurrentReindex).not_to receive(:new)
subject
end
end
it 'does not perform reindexing actions if lease is not granted' do
expect(lease).to receive(:try_obtain).ordered.and_return(false)
expect(Gitlab::Database::Reindexing::ConcurrentReindex).not_to receive(:new)
context 'notifications' do
it 'sends #notify_start before reindexing' do
expect(notifier).to receive(:notify_start).with(action).ordered
expect(reindexer).to receive(:perform).ordered
subject
subject
end
it 'sends #notify_end after reindexing and updating the action is done' do
expect(action).to receive(:finish).ordered
expect(notifier).to receive(:notify_end).with(action).ordered
subject
end
end
context 'action tracking' do
it 'calls #finish on the action' do
expect(reindexer).to receive(:perform).ordered
expect(action).to receive(:finish).ordered
subject
end
it 'upon error, it still calls finish and raises the error' do
expect(reindexer).to receive(:perform).ordered.and_raise('something went wrong')
expect(action).to receive(:finish).ordered
expect { subject }.to raise_error(/something went wrong/)
expect(action).to be_failed
end
end
end
context 'notifications' do
it 'sends #notify_start before reindexing' do
expect(notifier).to receive(:notify_start).with(action).ordered
expect(reindexer).to receive(:perform).ordered
context 'legacy reindexing method (< PG12) - to be removed' do
before do
stub_feature_flags(database_reindexing_pg12: false)
swapout_view_for_table(:postgres_indexes)
subject
allow(Gitlab::Database::Reindexing::ConcurrentReindex).to receive(:new).with(index).and_return(reindexer)
allow(Gitlab::Database::Reindexing::ReindexAction).to receive(:create_for).with(index).and_return(action)
end
it 'sends #notify_end after reindexing and updating the action is done' do
expect(action).to receive(:finish).ordered
expect(notifier).to receive(:notify_end).with(action).ordered
subject
end
it_behaves_like 'reindexing coordination'
end
context 'action tracking' do
it 'calls #finish on the action' do
expect(reindexer).to receive(:perform).ordered
expect(action).to receive(:finish).ordered
context 'PG12 reindexing method' do
before do
stub_feature_flags(database_reindexing_pg12: true)
swapout_view_for_table(:postgres_indexes)
subject
allow(Gitlab::Database::Reindexing::ReindexConcurrently).to receive(:new).with(index).and_return(reindexer)
allow(Gitlab::Database::Reindexing::ReindexAction).to receive(:create_for).with(index).and_return(action)
end
it 'upon error, it still calls finish and raises the error' do
expect(reindexer).to receive(:perform).ordered.and_raise('something went wrong')
expect(action).to receive(:finish).ordered
expect { subject }.to raise_error(/something went wrong/)
expect(action).to be_failed
end
it_behaves_like 'reindexing coordination'
end
end
end

134
spec/lib/gitlab/database/reindexing/reindex_concurrently_spec.rb Normal file
View File

@ -0,0 +1,134 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Database::Reindexing::ReindexConcurrently, '#perform' do
subject { described_class.new(index, logger: logger).perform }
let(:table_name) { '_test_reindex_table' }
let(:column_name) { '_test_column' }
let(:index_name) { '_test_reindex_index' }
let(:index) { Gitlab::Database::PostgresIndex.by_identifier("public.#{iname(index_name)}") }
let(:logger) { double('logger', debug: nil, info: nil, error: nil ) }
let(:connection) { ActiveRecord::Base.connection }
before do
connection.execute(<<~SQL)
CREATE TABLE #{table_name} (
id serial NOT NULL PRIMARY KEY,
#{column_name} integer NOT NULL);
CREATE INDEX #{index_name} ON #{table_name} (#{column_name});
SQL
end
context 'when the index serves an exclusion constraint' do
before do
allow(index).to receive(:exclusion?).and_return(true)
end
it 'raises an error' do
expect { subject }.to raise_error(described_class::ReindexError, /indexes serving an exclusion constraint are currently not supported/)
end
end
context 'when attempting to reindex an expression index' do
before do
allow(index).to receive(:expression?).and_return(true)
end
it 'raises an error' do
expect { subject }.to raise_error(described_class::ReindexError, /expression indexes are currently not supported/)
end
end
context 'when the index is a dangling temporary index from a previous reindexing run' do
context 'with the temporary index prefix' do
let(:index_name) { '_test_reindex_index_ccnew' }
it 'raises an error' do
expect { subject }.to raise_error(described_class::ReindexError, /left-over temporary index/)
end
end
context 'with the temporary index prefix with a counter' do
let(:index_name) { '_test_reindex_index_ccnew1' }
it 'raises an error' do
expect { subject }.to raise_error(described_class::ReindexError, /left-over temporary index/)
end
end
end
it 'recreates the index using REINDEX with a long statement timeout' do
expect_to_execute_in_order(
"SET statement_timeout TO '32400s'",
"REINDEX INDEX CONCURRENTLY \"public\".\"#{index.name}\"",
"RESET statement_timeout"
)
subject
end
context 'with dangling indexes matching TEMPORARY_INDEX_PATTERN, i.e. /some\_index\_ccnew(\d)*/' do
before do
# dangling indexes
connection.execute("CREATE INDEX #{iname(index_name, '_ccnew')} ON #{table_name} (#{column_name})")
connection.execute("CREATE INDEX #{iname(index_name, '_ccnew2')} ON #{table_name} (#{column_name})")
# Unrelated index - don't drop
connection.execute("CREATE INDEX some_other_index_ccnew ON #{table_name} (#{column_name})")
end
shared_examples_for 'dropping the dangling index' do
it 'drops the dangling indexes while controlling lock_timeout' do
expect_to_execute_in_order(
# Regular index rebuild
"SET statement_timeout TO '32400s'",
"REINDEX INDEX CONCURRENTLY \"public\".\"#{index_name}\"",
"RESET statement_timeout",
# Drop _ccnew index
"SET lock_timeout TO '60000ms'",
"DROP INDEX CONCURRENTLY IF EXISTS \"public\".\"#{iname(index_name, '_ccnew')}\"",
"RESET idle_in_transaction_session_timeout; RESET lock_timeout",
# Drop _ccnew2 index
"SET lock_timeout TO '60000ms'",
"DROP INDEX CONCURRENTLY IF EXISTS \"public\".\"#{iname(index_name, '_ccnew2')}\"",
"RESET idle_in_transaction_session_timeout; RESET lock_timeout"
)
subject
end
end
context 'with normal index names' do
it_behaves_like 'dropping the dangling index'
end
context 'with index name at 63 character limit' do
let(:index_name) { 'a' * 63 }
before do
# Another unrelated index - don't drop
extra_index = index_name[0...55]
connection.execute("CREATE INDEX #{extra_index}_ccnew ON #{table_name} (#{column_name})")
end
it_behaves_like 'dropping the dangling index'
end
end
def iname(name, suffix = '')
"#{name[0...63 - suffix.size]}#{suffix}"
end
def expect_to_execute_in_order(*queries)
# Indexes cannot be created CONCURRENTLY in a transaction. Since the tests are wrapped in transactions,
# verify the original call but pass through the non-concurrent form.
queries.each do |query|
expect(connection).to receive(:execute).with(query).ordered.and_wrap_original do |method, sql|
method.call(sql.sub(/CONCURRENTLY/, ''))
end
end
end
end

27
spec/lib/gitlab/database/reindexing_spec.rb
View File

@ -29,11 +29,30 @@ RSpec.describe Gitlab::Database::Reindexing do
describe '.candidate_indexes' do
subject { described_class.candidate_indexes }
it 'retrieves regular indexes that are no left-overs from previous runs' do
result = double
expect(Gitlab::Database::PostgresIndex).to receive_message_chain('regular.where.not_match.not_match').with(no_args).with('NOT expression').with('^tmp_reindex_').with('^old_reindex_').and_return(result)
context 'with deprecated method for < PG12' do
before do
stub_feature_flags(database_reindexing_pg12: false)
end
expect(subject).to eq(result)
it 'retrieves regular indexes that are no left-overs from previous runs' do
result = double
expect(Gitlab::Database::PostgresIndex).to receive_message_chain('not_match.not_match.not_match.regular').with('^tmp_reindex_').with('^old_reindex_').with('\_ccnew[0-9]*$').with(no_args).and_return(result)
expect(subject).to eq(result)
end
end
context 'with deprecated method for >= PG12' do
before do
stub_feature_flags(database_reindexing_pg12: true)
end
it 'retrieves regular indexes that are no left-overs from previous runs' do
result = double
expect(Gitlab::Database::PostgresIndex).to receive_message_chain('not_match.not_match.not_match.reindexing_support').with('^tmp_reindex_').with('^old_reindex_').with('\_ccnew[0-9]*$').with(no_args).and_return(result)
expect(subject).to eq(result)
end
end
end
end