Add latest changes from gitlab-org/gitlab@master

db/migrate/20240116205943_add_elasticsearch_max_code_indexing_concurrency_to_application_settings.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class AddElasticsearchMaxCodeIndexingConcurrencyToApplicationSettings < Gitlab::Database::Migration[2.2]
+  enable_lock_retries!
+  milestone '16.9'
+
+  def change
+    add_column :application_settings,
+      :elasticsearch_max_code_indexing_concurrency,
+      :integer,
+      default: 30,
+      null: false,
+      if_not_exists: true
+  end
+end

db/migrate/20240116211138_update_max_code_indexing_concurrency_in_application_settings_for_gitlab_com.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class UpdateMaxCodeIndexingConcurrencyInApplicationSettingsForGitlabCom < Gitlab::Database::Migration[2.2]
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+  enable_lock_retries!
+
+  milestone '16.9'
+
+  def up
+    return unless Gitlab.com?
+
+    execute 'UPDATE application_settings SET elasticsearch_max_code_indexing_concurrency = 60'
+  end
+
+  def down
+    return unless Gitlab.com?
+
+    execute 'UPDATE application_settings SET elasticsearch_max_code_indexing_concurrency = 30'
+  end
+end

db/schema_migrations/20240116205943

@@ -0,0 +1 @@
+f86c644bc630c52bf8647576f391b7a77d4b0d9a13a49c7552d0ec68f59c21c8

db/schema_migrations/20240116211138

@@ -0,0 +1 @@
+72a612645aea55077e46a0c99860312f677e03fb9956cfd13cee61a4bfa53faa

db/structure.sql

@@ -12632,6 +12632,7 @@ CREATE TABLE application_settings (
     toggle_security_policies_policy_scope boolean DEFAULT false NOT NULL,
     lock_toggle_security_policies_policy_scope boolean DEFAULT false NOT NULL,
     rate_limits jsonb DEFAULT '{}'::jsonb NOT NULL,
+    elasticsearch_max_code_indexing_concurrency integer DEFAULT 30 NOT NULL,
     CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)),
     CONSTRAINT app_settings_container_registry_pre_import_tags_rate_positive CHECK ((container_registry_pre_import_tags_rate >= (0)::numeric)),
     CONSTRAINT app_settings_dep_proxy_ttl_policies_worker_capacity_positive CHECK ((dependency_proxy_ttl_group_policy_worker_capacity >= 0)),

doc/api/settings.md

@@ -402,6 +402,7 @@ listed in the descriptions of the relevant settings.
 | `elasticsearch_requeue_workers` **(PREMIUM ALL)**            | boolean           | no                  | Enable automatic requeuing of indexing workers. This improves non-code indexing throughput by enqueuing Sidekiq jobs until all documents are processed. |
 | `elasticsearch_limit_indexing` **(PREMIUM ALL)**             | boolean           | no                  | Limit Elasticsearch to index certain namespaces and projects. |
 | `elasticsearch_max_bulk_concurrency` **(PREMIUM ALL)**       | integer           | no                  | Maximum concurrency of Elasticsearch bulk requests per indexing operation. This only applies to repository indexing operations. |
+| `elasticsearch_max_code_indexing_concurrency` **(PREMIUM ALL)**       | integer  | no                  | Maximum concurrency of Elasticsearch code indexing background jobs. This only applies to repository indexing operations. |
 | `elasticsearch_worker_number_of_shards` **(PREMIUM ALL)**    | integer           | no                  | Number of indexing worker shards. This improves non-code indexing throughput by enqueuing more parallel Sidekiq jobs. Default is `2`. |
 | `elasticsearch_max_bulk_size_mb` **(PREMIUM ALL)**           | integer           | no                  | Maximum size of Elasticsearch bulk indexing requests in MB. This only applies to repository indexing operations. |
 | `elasticsearch_namespace_ids` **(PREMIUM ALL)**              | array of integers | no                  | The namespaces to index via Elasticsearch if `elasticsearch_limit_indexing` is enabled. |

locale/gitlab.pot

@@ -12038,6 +12038,9 @@ msgstr ""
 msgid "Code coverage statistics for %{ref} %{start_date} - %{end_date}"
 msgstr ""
 
+msgid "Code indexing concurrency"
+msgstr ""
+
 msgid "Code owner approval is required"
 msgstr ""
 
@@ -24561,6 +24564,9 @@ msgstr ""
 msgid "How is progress calculated?"
 msgstr ""
 
+msgid "How many code indexing jobs are allowed to run concurrently. A higher value improves repository indexing performance by increasing Sidekiq and Elasticsearch load. Setting it to 0 allows an unlimited number of indexing jobs."
+msgstr ""
+
 msgid "How many seconds an IP counts toward the IP address limit."
 msgstr ""
 

spec/migrations/20240116211138_update_max_code_indexing_concurrency_in_application_settings_for_gitlab_com_spec.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe UpdateMaxCodeIndexingConcurrencyInApplicationSettingsForGitlabCom, feature_category: :global_search do
+  let(:settings) { table(:application_settings) }
+
+  describe "#up" do
+    subject(:up) { migrate! }
+
+    it 'does nothing when not in gitlab.com' do
+      record = settings.create!
+
+      expect { up }.not_to change { record.reload.elasticsearch_max_code_indexing_concurrency }
+    end
+
+    it 'updates elasticsearch_worker_number_of_shards when gitlab.com' do
+      allow(Gitlab).to receive(:com?).and_return(true)
+
+      record = settings.create!
+
+      expect { up }.to change { record.reload.elasticsearch_max_code_indexing_concurrency }.from(30).to(60)
+    end
+  end
+
+  describe "#down" do
+    subject(:down) { schema_migrate_down! }
+
+    it 'does nothing when not in gitlab.com' do
+      record = settings.create!(elasticsearch_max_code_indexing_concurrency: 60)
+
+      migrate!
+
+      expect { down }.not_to change { record.reload.elasticsearch_max_code_indexing_concurrency }
+    end
+
+    it 'updates elasticsearch_worker_number_of_shards when gitlab.com' do
+      allow(Gitlab).to receive(:com?).and_return(true)
+
+      record = settings.create!(elasticsearch_max_code_indexing_concurrency: 60)
+
+      migrate!
+
+      expect { down }.to change { record.reload.elasticsearch_max_code_indexing_concurrency }.from(60).to(30)
+    end
+  end
+end