diff --git a/.rubocop_todo/rspec/excessive_docstring_spacing.yml b/.rubocop_todo/rspec/excessive_docstring_spacing.yml index 0a42ac3f39d..c36ff14e096 100644 --- a/.rubocop_todo/rspec/excessive_docstring_spacing.yml +++ b/.rubocop_todo/rspec/excessive_docstring_spacing.yml @@ -8,7 +8,6 @@ RSpec/ExcessiveDocstringSpacing: - 'ee/spec/finders/ee/issuables/label_filter_spec.rb' - 'ee/spec/finders/ee/namespaces/projects_finder_spec.rb' - 'ee/spec/graphql/mutations/ai/action_spec.rb' - - 'ee/spec/helpers/ee/members_helper_spec.rb' - 'ee/spec/helpers/tree_helper_spec.rb' - 'ee/spec/lib/ee/gitlab/ci/config/entry/bridge_spec.rb' - 'ee/spec/lib/elastic/latest/git_class_proxy_spec.rb' diff --git a/Gemfile.checksum b/Gemfile.checksum index b2f219d0a7e..0a3dc011f68 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -226,11 +226,11 @@ {"name":"gitlab-dangerfiles","version":"4.8.0","platform":"ruby","checksum":"b327d079552ec974a63bf34d749a0308425af6ebf51d01064f1a6ff216a523db"}, {"name":"gitlab-experiment","version":"0.9.1","platform":"ruby","checksum":"f230ee742154805a755d5f2539dc44d93cdff08c5bbbb7656018d61f93d01f48"}, {"name":"gitlab-fog-azure-rm","version":"2.2.0","platform":"ruby","checksum":"31aa7c2170f57874053144e7f716ec9e15f32e71ffbd2c56753dce46e2e78ba9"}, -{"name":"gitlab-glfm-markdown","version":"0.0.21","platform":"aarch64-linux","checksum":"e2bea2e58b4553fc908d9bf947beafb977a639868f68df52eec5b2a2036ebb9e"}, -{"name":"gitlab-glfm-markdown","version":"0.0.21","platform":"arm64-darwin","checksum":"85668cb0cb8e361e40682899fab76df1c623540cab561489f6a8e057a3cbaf8b"}, -{"name":"gitlab-glfm-markdown","version":"0.0.21","platform":"ruby","checksum":"cb960ac1bc509d72b460c9dc934fb0a02cf061a5de6b1b00c72b794817d63b40"}, -{"name":"gitlab-glfm-markdown","version":"0.0.21","platform":"x86_64-darwin","checksum":"8425ee27e0b32b75619e08e1700c1302297b44928adc19a026bea243c96363f5"}, -{"name":"gitlab-glfm-markdown","version":"0.0.21","platform":"x86_64-linux","checksum":"9ea7d7a7a20c15960839521459a82edab787a4d8475ee412beba8362aa5fcd71"}, +{"name":"gitlab-glfm-markdown","version":"0.0.23","platform":"aarch64-linux","checksum":"2debf90c2d7b03e282a88951ad39a8f9bfc1662be7329f6dbc66c56cf9f2c17a"}, +{"name":"gitlab-glfm-markdown","version":"0.0.23","platform":"arm64-darwin","checksum":"e1d5fe80b52263041c1e91d85b4fd9eb367c20e2bcda817132bfcf671ea1874a"}, +{"name":"gitlab-glfm-markdown","version":"0.0.23","platform":"ruby","checksum":"89a12909c39aea326adb0b7194f7b89d61b4f9122308435fba0bcb84e4f4ff24"}, +{"name":"gitlab-glfm-markdown","version":"0.0.23","platform":"x86_64-darwin","checksum":"4b77a37358d98c3b2269f7dd19f6549555c5de00bf12a4eca25c34076f72f78d"}, +{"name":"gitlab-glfm-markdown","version":"0.0.23","platform":"x86_64-linux","checksum":"2b71ec5ae06a524114e2cf423ce6635fd1f5c6776c0c956188aa0b2f0fbfbead"}, {"name":"gitlab-kas-grpc","version":"17.5.1","platform":"ruby","checksum":"88639bfaa9301d78a7fbff696ec262ed696a15a6f41c1b51bffe6b39c7a61ca7"}, {"name":"gitlab-labkit","version":"0.37.0","platform":"ruby","checksum":"d2dd0a60db2149a9a8eebf2975dc23f54ac3ceb01bdba732eb1b26b86dfffa70"}, {"name":"gitlab-license","version":"2.6.0","platform":"ruby","checksum":"2c1f8ae73835640ec77bf758c1d0c9730635043c01cf77902f7976e826d7d016"}, diff --git a/Gemfile.lock b/Gemfile.lock index 81230308dcd..948fdb6e84a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -745,7 +745,7 @@ GEM mime-types net-http-persistent (~> 4.0) nokogiri (~> 1, >= 1.10.8) - gitlab-glfm-markdown (0.0.21) + gitlab-glfm-markdown (0.0.23) rb_sys (= 0.9.94) gitlab-kas-grpc (17.5.1) grpc (~> 1.0) diff --git a/Gemfile.next.checksum b/Gemfile.next.checksum index 38aae54493b..bc2b55913e6 100644 --- a/Gemfile.next.checksum +++ b/Gemfile.next.checksum @@ -227,11 +227,11 @@ {"name":"gitlab-dangerfiles","version":"4.8.0","platform":"ruby","checksum":"b327d079552ec974a63bf34d749a0308425af6ebf51d01064f1a6ff216a523db"}, {"name":"gitlab-experiment","version":"0.9.1","platform":"ruby","checksum":"f230ee742154805a755d5f2539dc44d93cdff08c5bbbb7656018d61f93d01f48"}, {"name":"gitlab-fog-azure-rm","version":"2.2.0","platform":"ruby","checksum":"31aa7c2170f57874053144e7f716ec9e15f32e71ffbd2c56753dce46e2e78ba9"}, -{"name":"gitlab-glfm-markdown","version":"0.0.21","platform":"aarch64-linux","checksum":"e2bea2e58b4553fc908d9bf947beafb977a639868f68df52eec5b2a2036ebb9e"}, -{"name":"gitlab-glfm-markdown","version":"0.0.21","platform":"arm64-darwin","checksum":"85668cb0cb8e361e40682899fab76df1c623540cab561489f6a8e057a3cbaf8b"}, -{"name":"gitlab-glfm-markdown","version":"0.0.21","platform":"ruby","checksum":"cb960ac1bc509d72b460c9dc934fb0a02cf061a5de6b1b00c72b794817d63b40"}, -{"name":"gitlab-glfm-markdown","version":"0.0.21","platform":"x86_64-darwin","checksum":"8425ee27e0b32b75619e08e1700c1302297b44928adc19a026bea243c96363f5"}, -{"name":"gitlab-glfm-markdown","version":"0.0.21","platform":"x86_64-linux","checksum":"9ea7d7a7a20c15960839521459a82edab787a4d8475ee412beba8362aa5fcd71"}, +{"name":"gitlab-glfm-markdown","version":"0.0.23","platform":"aarch64-linux","checksum":"2debf90c2d7b03e282a88951ad39a8f9bfc1662be7329f6dbc66c56cf9f2c17a"}, +{"name":"gitlab-glfm-markdown","version":"0.0.23","platform":"arm64-darwin","checksum":"e1d5fe80b52263041c1e91d85b4fd9eb367c20e2bcda817132bfcf671ea1874a"}, +{"name":"gitlab-glfm-markdown","version":"0.0.23","platform":"ruby","checksum":"89a12909c39aea326adb0b7194f7b89d61b4f9122308435fba0bcb84e4f4ff24"}, +{"name":"gitlab-glfm-markdown","version":"0.0.23","platform":"x86_64-darwin","checksum":"4b77a37358d98c3b2269f7dd19f6549555c5de00bf12a4eca25c34076f72f78d"}, +{"name":"gitlab-glfm-markdown","version":"0.0.23","platform":"x86_64-linux","checksum":"2b71ec5ae06a524114e2cf423ce6635fd1f5c6776c0c956188aa0b2f0fbfbead"}, {"name":"gitlab-kas-grpc","version":"17.5.1","platform":"ruby","checksum":"88639bfaa9301d78a7fbff696ec262ed696a15a6f41c1b51bffe6b39c7a61ca7"}, {"name":"gitlab-labkit","version":"0.37.0","platform":"ruby","checksum":"d2dd0a60db2149a9a8eebf2975dc23f54ac3ceb01bdba732eb1b26b86dfffa70"}, {"name":"gitlab-license","version":"2.6.0","platform":"ruby","checksum":"2c1f8ae73835640ec77bf758c1d0c9730635043c01cf77902f7976e826d7d016"}, diff --git a/Gemfile.next.lock b/Gemfile.next.lock index dafb78ca3b3..422ed67c64d 100644 --- a/Gemfile.next.lock +++ b/Gemfile.next.lock @@ -755,7 +755,7 @@ GEM mime-types net-http-persistent (~> 4.0) nokogiri (~> 1, >= 1.10.8) - gitlab-glfm-markdown (0.0.21) + gitlab-glfm-markdown (0.0.23) rb_sys (= 0.9.94) gitlab-kas-grpc (17.5.1) grpc (~> 1.0) diff --git a/app/assets/javascripts/glql/components/common/facade.vue b/app/assets/javascripts/glql/components/common/facade.vue index b42b5caf43c..bd40a87695c 100644 --- a/app/assets/javascripts/glql/components/common/facade.vue +++ b/app/assets/javascripts/glql/components/common/facade.vue @@ -133,7 +133,7 @@ export default { />
{{ query.trim() }}
+ >{{ query.trim() }} diff --git a/app/assets/javascripts/search/sidebar/components/app.vue b/app/assets/javascripts/search/sidebar/components/app.vue index 075b22575a3..35d3ff5b67a 100644 --- a/app/assets/javascripts/search/sidebar/components/app.vue +++ b/app/assets/javascripts/search/sidebar/components/app.vue @@ -2,6 +2,7 @@ // eslint-disable-next-line no-restricted-imports import { mapState, mapGetters } from 'vuex'; import { __ } from '~/locale'; +import * as Sentry from '~/sentry/sentry_browser_wrapper'; import ScopeSidebarNavigation from '~/search/sidebar/components/scope_sidebar_navigation.vue'; import SidebarPortal from '~/super_sidebar/components/sidebar_portal.vue'; import { toggleSuperSidebarCollapsed } from '~/super_sidebar/super_sidebar_collapsed_state_manager'; @@ -84,6 +85,11 @@ export default { return this.currentScope === SCOPE_WIKI_BLOBS; }, }, + beforeCreate() { + if (!this.$store) { + Sentry.captureException('GlobalSearchSidebar was not provided a Vuex store'); + } + }, methods: { toggleFiltersFromSidebar() { toggleSuperSidebarCollapsed(); diff --git a/app/assets/javascripts/search/sidebar/components/author_filter/index.vue b/app/assets/javascripts/search/sidebar/components/author_filter/index.vue new file mode 100644 index 00000000000..00db2c442e5 --- /dev/null +++ b/app/assets/javascripts/search/sidebar/components/author_filter/index.vue @@ -0,0 +1,170 @@ + + + diff --git a/app/assets/javascripts/search/sidebar/components/merge_requests_filters.vue b/app/assets/javascripts/search/sidebar/components/merge_requests_filters.vue index cb8281324a2..6ca28010aaf 100644 --- a/app/assets/javascripts/search/sidebar/components/merge_requests_filters.vue +++ b/app/assets/javascripts/search/sidebar/components/merge_requests_filters.vue @@ -8,6 +8,7 @@ import FiltersTemplate from './filters_template.vue'; import LabelFilter from './label_filter/index.vue'; import ArchivedFilter from './archived_filter/index.vue'; import SourceBranchFilter from './source_branch_filter/index.vue'; +import AuthorFilter from './author_filter/index.vue'; export default { name: 'MergeRequestsFilters', @@ -17,6 +18,7 @@ export default { LabelFilter, ArchivedFilter, SourceBranchFilter, + AuthorFilter, }, mixins: [glFeatureFlagsMixin()], computed: { @@ -28,9 +30,12 @@ export default { (!this.hasMissingProjectContext || this.groupInitialJson?.id) ); }, - shouldShowLabelFilter() { + isAdvancedSearch() { return this.searchType === SEARCH_TYPE_ADVANCED; }, + shouldShowAuthorFilter() { + return this.isAdvancedSearch && this.glFeatures.searchMrFilterAuthor; + }, }, }; @@ -38,8 +43,9 @@ export default { diff --git a/app/assets/javascripts/search/sidebar/components/shared/filter_dropdown.vue b/app/assets/javascripts/search/sidebar/components/shared/filter_dropdown.vue index 87d2e735f05..ce7fea111f3 100644 --- a/app/assets/javascripts/search/sidebar/components/shared/filter_dropdown.vue +++ b/app/assets/javascripts/search/sidebar/components/shared/filter_dropdown.vue @@ -17,10 +17,10 @@ export default { type: Array, required: true, }, - errors: { - type: Array, + error: { + type: String, required: false, - default: () => [], + default: '', }, headerText: { type: String, @@ -57,9 +57,7 @@ export default { }, data() { return { - selectedRef: '', query: '', - hasError: this.errors.length > 0, }; }, computed: { @@ -83,6 +81,9 @@ export default { ? this.$options.i18n.noSearchResultsText : this.$options.i18n.noLoadResultsText; }, + hasError() { + return Boolean(this.error); + }, }, created() { this.debouncedSearch = debounce(this.search, DEFAULT_DEBOUNCE_AND_THROTTLE_MS); @@ -104,8 +105,8 @@ export default { } this.searchResults = fuzzaldrinPlus.filter(this.listData, this.query, { key: ['text'] }); }, - selectRef(ref) { - this.$emit('selected', ref); + selectRef(selectedAuthorValue) { + this.$emit('selected', selectedAuthorValue); }, onHide() { if (!this.query || this.searchResults.length > 0) { @@ -149,13 +150,12 @@ export default { diff --git a/app/assets/javascripts/search/sidebar/components/source_branch_filter/index.vue b/app/assets/javascripts/search/sidebar/components/source_branch_filter/index.vue index 14f14c734c8..62e11f220de 100644 --- a/app/assets/javascripts/search/sidebar/components/source_branch_filter/index.vue +++ b/app/assets/javascripts/search/sidebar/components/source_branch_filter/index.vue @@ -2,6 +2,7 @@ // eslint-disable-next-line no-restricted-imports import { mapActions, mapState } from 'vuex'; import { GlFormCheckbox, GlTooltipDirective } from '@gitlab/ui'; +import * as Sentry from '~/sentry/sentry_browser_wrapper'; import { s__ } from '~/locale'; import AjaxCache from '~/lib/utils/ajax_cache'; import { mergeUrlParams } from '~/lib/utils/url_utility'; @@ -31,7 +32,7 @@ export default { data() { return { sourceBranches: [], - errors: [], + error: '', toggleState: false, selectedBranch: '', isLoading: false, @@ -74,12 +75,13 @@ export default { this.isLoading = true; try { const data = await AjaxCache.retrieve(this.getMergeRequestSourceBranchesEndpoint()); - this.errors = []; + this.error = ''; this.isLoading = false; this.sourceBranches = this.convertToListboxItems(data); - } catch (e) { + } catch (error) { + Sentry.captureException(error); this.isLoading = false; - this.errors.push(e.message); + this.error = error.message; } }, handleSelected(ref) { @@ -122,7 +124,7 @@ export default { (status) { where(status: status) } scope :created_after, ->(time) { where(arel_table[:created_at].gt(time)) } + scope :created_before, ->(time) { where(arel_table[:created_at].lt(time)) } scope :created_before_id, ->(id) { where(arel_table[:id].lt(id)) } scope :before_pipeline, ->(pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) } scope :with_pipeline_source, ->(source) { where(source: source) } diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index e59915e7629..d7263c4d92f 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -13,7 +13,6 @@ module Ci include TaggableQueries include Presentable include EachBatch - include Ci::HasRunnerExecutor include Ci::HasRunnerStatus include Ci::Taggable @@ -88,6 +87,7 @@ module Ci has_many :projects, through: :runner_projects, disable_joins: true has_many :runner_namespaces, inverse_of: :runner, autosave: true has_many :groups, through: :runner_namespaces, disable_joins: true + has_many :tag_links, class_name: 'Ci::RunnerTagging', inverse_of: :runner # currently we have only 1 namespace assigned, but order is here for consistency has_one :owner_runner_namespace, -> { order(:id) }, class_name: 'Ci::RunnerNamespace' @@ -330,6 +330,14 @@ module Ci end end + # TODO: Remove once https://gitlab.com/gitlab-org/gitlab/-/issues/504277 is closed. + def self.sharded_table_proxy_model + @sharded_table_proxy_class ||= Class.new(self) do + self.table_name = :ci_runners_e59bb2812d + self.primary_key = :id + end + end + def runner_matcher Gitlab::Ci::Matching::RunnerMatcher.new({ runner_ids: [id], @@ -438,6 +446,44 @@ module Ci tag_list.any? end + override :save_tags + def save_tags + super do |new_tags, old_tags| + next if ::Feature.disabled?(:write_to_ci_runner_taggings, owner) + + if old_tags.present? + tag_links + .where(tag_id: old_tags) + .delete_all + end + + # Avoid inserting partitioned taggings that refer to a missing ci_runners partitioned record, since + # the backfill is not yet finalized. + ensure_partitioned_runner_record_exists if new_tags.any? + + ci_runner_taggings = new_tags.map do |tag| + Ci::RunnerTagging.new( + runner_id: id, runner_type: runner_type, + tag_id: tag.id, sharding_key_id: sharding_key_id) + end + + ::Ci::RunnerTagging.bulk_insert!( + ci_runner_taggings, + validate: false, + unique_by: [:tag_id, :runner_id, :runner_type], + returns: :id + ) + end + end + + # TODO: Remove once https://gitlab.com/gitlab-org/gitlab/-/issues/504277 is closed. + def ensure_partitioned_runner_record_exists + self.class.sharded_table_proxy_model.insert_all( + [attributes.except('tag_list')], unique_by: [:id, :runner_type], + returning: false, record_timestamps: false + ) + end + def predefined_variables Gitlab::Ci::Variables::Collection.new .append(key: 'CI_RUNNER_ID', value: id.to_s) diff --git a/app/models/ci/runner_tagging.rb b/app/models/ci/runner_tagging.rb index 0e1037756b9..75b1b4c050c 100644 --- a/app/models/ci/runner_tagging.rb +++ b/app/models/ci/runner_tagging.rb @@ -2,6 +2,8 @@ module Ci class RunnerTagging < Ci::ApplicationRecord + include BulkInsertSafe + self.table_name = :ci_runner_taggings self.primary_key = :id diff --git a/app/models/project_ci_cd_setting.rb b/app/models/project_ci_cd_setting.rb index 0d91e879017..27f017c47fc 100644 --- a/app/models/project_ci_cd_setting.rb +++ b/app/models/project_ci_cd_setting.rb @@ -50,8 +50,15 @@ class ProjectCiCdSetting < ApplicationRecord chronic_duration_attr :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval chronic_duration_attr_writer :delete_pipelines_in_human_readable, :delete_pipelines_in_seconds - scope :configured_to_delete_old_pipelines, -> { where.not(delete_pipelines_in_seconds: nil) } - scope :with_project, -> { preload(:project) } + scope :for_project, ->(ids) { where(project_id: ids) } + scope :order_project_id_asc, -> { order(project_id: :asc) } + scope :configured_to_delete_old_pipelines, -> do + where.not(delete_pipelines_in_seconds: nil) + end + + def self.pluck_project_id(limit) + limit(limit).pluck(:project_id) + end def keep_latest_artifacts_available? # The project level feature can only be enabled when the feature is enabled instance wide diff --git a/app/models/users/credit_card_validation.rb b/app/models/users/credit_card_validation.rb index 31f368de41c..8db0a3efc7c 100644 --- a/app/models/users/credit_card_validation.rb +++ b/app/models/users/credit_card_validation.rb @@ -82,8 +82,6 @@ module Users end def exceeded_daily_verification_limit? - return false unless Feature.enabled?(:credit_card_validation_daily_limit, user, type: :gitlab_com_derisk) - duplicate_record_count = self.class .where(stripe_card_fingerprint: stripe_card_fingerprint) .where('credit_card_validated_at > ?', 24.hours.ago) diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 8cc96201f47..9aa1618f3b2 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -336,6 +336,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: cronjob:ci_schedule_old_pipelines_removal_cron + :worker_name: Ci::ScheduleOldPipelinesRemovalCronWorker + :feature_category: :continuous_integration + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:ci_schedule_unlock_pipelines_in_queue_cron :worker_name: Ci::ScheduleUnlockPipelinesInQueueCronWorker :feature_category: :job_artifacts @@ -2901,6 +2910,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: ci_destroy_old_pipelines + :worker_name: Ci::DestroyOldPipelinesWorker + :feature_category: :continuous_integration + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: ci_initialize_pipelines_iid_sequence :worker_name: Ci::InitializePipelinesIidSequenceWorker :feature_category: :continuous_integration diff --git a/app/workers/ci/destroy_old_pipelines_worker.rb b/app/workers/ci/destroy_old_pipelines_worker.rb new file mode 100644 index 00000000000..366f3e59e89 --- /dev/null +++ b/app/workers/ci/destroy_old_pipelines_worker.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Ci + class DestroyOldPipelinesWorker + include ApplicationWorker + include LimitedCapacity::Worker + + data_consistency :sticky + feature_category :continuous_integration + urgency :low + idempotent! + + LIMIT = 250 + CONCURRENCY = 10 + + def perform_work(*) + Project.find_by_id(fetch_next_project_id).try do |project| + with_context(project: project) do + timestamp = project.ci_delete_pipelines_in_seconds.seconds.ago + pipelines = Ci::Pipeline.for_project(project.id).created_before(timestamp).limit(LIMIT).to_a + pipelines.each { |pipeline| Ci::DestroyPipelineService.new(project, nil).unsafe_execute(pipeline) } + end + end + end + + def max_running_jobs + CONCURRENCY + end + + def remaining_work_count(*) + Gitlab::Redis::SharedState.with do |redis| + redis.scard(queue_key) + end + end + + private + + def fetch_next_project_id + Gitlab::Redis::SharedState.with do |redis| + redis.lpop(queue_key) + end + end + + def queue_key + Ci::ScheduleOldPipelinesRemovalCronWorker::QUEUE_KEY + end + end +end diff --git a/app/workers/ci/schedule_old_pipelines_removal_cron_worker.rb b/app/workers/ci/schedule_old_pipelines_removal_cron_worker.rb new file mode 100644 index 00000000000..4d9dd281aac --- /dev/null +++ b/app/workers/ci/schedule_old_pipelines_removal_cron_worker.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Ci + class ScheduleOldPipelinesRemovalCronWorker + include ApplicationWorker + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext -- does not perform work scoped to a context + + urgency :low + idempotent! + deduplicate :until_executed, including_scheduled: true + feature_category :continuous_integration + data_consistency :sticky + + PROJECTS_LIMIT = 1_000 + LAST_PROCESSED_REDIS_KEY = 'ci_old_pipelines_removal_last_processed_project_id{}' + REDIS_EXPIRATION_TIME = 2.hours.to_i + QUEUE_KEY = 'ci_old_pipelines_removal_project_ids_queue{}' + + def perform + return if Feature.disabled?(:ci_delete_old_pipelines, :instance, type: :beta) + + limit = PROJECTS_LIMIT - queued_entries_count + project_ids = fetch_next_project_ids(limit) + queue_projects_for_processing(project_ids) + remove_last_processed_id if project_ids.empty? || project_ids.size < limit + + Ci::DestroyOldPipelinesWorker.perform_with_capacity + end + + private + + def fetch_next_project_ids(limit) + ProjectCiCdSetting + .configured_to_delete_old_pipelines + .for_project(last_processed_id..) + .order_project_id_asc + .pluck_project_id(limit) + end + + def queued_entries_count + with_redis do |redis| + redis.scard(QUEUE_KEY).to_i + end + end + + def queue_projects_for_processing(ids) + return if ids.empty? + + with_redis do |redis| + redis.pipelined do |pipeline| + pipeline.rpush(QUEUE_KEY, ids) + pipeline.set(LAST_PROCESSED_REDIS_KEY, ids.last, ex: REDIS_EXPIRATION_TIME) + end + end + end + + def last_processed_id + with_redis do |redis| + redis.get(LAST_PROCESSED_REDIS_KEY).to_i + end + end + + def remove_last_processed_id + with_redis do |redis| + redis.del(LAST_PROCESSED_REDIS_KEY) + end + end + + def with_redis(&) + Gitlab::Redis::SharedState.with(&) # rubocop:disable CodeReuse/ActiveRecord -- not AR + end + end +end diff --git a/config/feature_flags/gitlab_com_derisk/credit_card_validation_daily_limit.yml b/config/feature_flags/gitlab_com_derisk/credit_card_validation_daily_limit.yml deleted file mode 100644 index 302f919c9a2..00000000000 --- a/config/feature_flags/gitlab_com_derisk/credit_card_validation_daily_limit.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: credit_card_validation_daily_limit -feature_issue_url: https://gitlab.com/gitlab-org/modelops/anti-abuse/team-tasks/-/issues/742 -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/159151 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/472122 -milestone: '17.3' -group: group::anti-abuse -type: gitlab_com_derisk -default_enabled: false diff --git a/config/feature_flags/gitlab_com_derisk/write_to_ci_runner_taggings.yml b/config/feature_flags/gitlab_com_derisk/write_to_ci_runner_taggings.yml new file mode 100644 index 00000000000..2ebeb5992b9 --- /dev/null +++ b/config/feature_flags/gitlab_com_derisk/write_to_ci_runner_taggings.yml @@ -0,0 +1,9 @@ +--- +name: write_to_ci_runner_taggings +feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/472974 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/173007 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/505003 +milestone: '17.7' +group: group::runner +type: gitlab_com_derisk +default_enabled: false diff --git a/config/feature_flags/wip/observability_features.yml b/config/feature_flags/wip/observability_features.yml index d878d66174b..19e34c81b92 100644 --- a/config/feature_flags/wip/observability_features.yml +++ b/config/feature_flags/wip/observability_features.yml @@ -4,5 +4,5 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/158786 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/472815 milestone: '17.3' type: wip -group: group::observability +group: group::platform insights default_enabled: false \ No newline at end of file diff --git a/config/gitlab_loose_foreign_keys.yml b/config/gitlab_loose_foreign_keys.yml index 6c4ef906dbe..dfc25af1259 100644 --- a/config/gitlab_loose_foreign_keys.yml +++ b/config/gitlab_loose_foreign_keys.yml @@ -3,6 +3,10 @@ abuse_report_assignees: - table: users column: user_id on_delete: async_delete +ai_conversation_messages: + - table: ai_conversation_threads + column: thread_id + on_delete: async_delete application_settings: - table: push_rules column: push_rule_id diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 38e2132f6bc..a36cfc80bf4 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -736,6 +736,9 @@ Settings.cron_jobs['database_monitor_locked_tables_cron_worker']['job_class'] = Settings.cron_jobs['merge_requests_process_scheduled_merge'] ||= {} Settings.cron_jobs['merge_requests_process_scheduled_merge']['cron'] ||= '*/1 * * * *' Settings.cron_jobs['merge_requests_process_scheduled_merge']['job_class'] = 'MergeRequests::ProcessScheduledMergeWorker' +Settings.cron_jobs['ci_schedule_old_pipelines_removal_cron_worker'] ||= {} +Settings.cron_jobs['ci_schedule_old_pipelines_removal_cron_worker']['cron'] ||= '*/11 * * * *' +Settings.cron_jobs['ci_schedule_old_pipelines_removal_cron_worker']['job_class'] = 'Ci::ScheduleOldPipelinesRemovalCronWorker' Gitlab.ee do Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker'] ||= {} diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 5303f781036..10e759923cc 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -179,6 +179,8 @@ - 1 - - ci_delete_objects - 1 +- - ci_destroy_old_pipelines + - 1 - - ci_initialize_pipelines_iid_sequence - 1 - - ci_job_artifacts_expire_project_build_artifacts diff --git a/data/deprecations/18-0-deprecate-gitaly-rate-limiters.yml b/data/deprecations/18-0-deprecate-gitaly-rate-limiters.yml new file mode 100644 index 00000000000..19f5efc5d9f --- /dev/null +++ b/data/deprecations/18-0-deprecate-gitaly-rate-limiters.yml @@ -0,0 +1,26 @@ +- title: "Gitaly rate limiting" + removal_milestone: "18.0" + announcement_milestone: "17.7" + breaking_change: false + reporter: qmnguyen0711 + stage: systems + issue_url: https://gitlab.com/gitlab-org/gitaly/-/issues/5011 + impact: low + scope: instance + resolution_role: Admin + manual_task: false + body: | + Because of the highly variable nature of Git operations and repository latencies, Gitaly + [RPC-based rate limiting](https://docs.gitlab.com/ee/administration/gitaly/monitoring.html#monitor-gitaly-rate-limiting) + is ineffective. Configuring proper rate limits is challenging and often becomes obsolete quickly because harmful + actions rarely generate enough requests per second to stand out. + + Gitaly already supports [concurrency limiting](https://docs.gitlab.com/ee/administration/gitaly/concurrency_limiting.html) and an + [adaptive limiting add-on](https://docs.gitlab.com/ee/administration/gitaly/concurrency_limiting.html#adaptive-concurrency-limiting), + which have proven to work well in production. + + Because Gitaly is not directly exposed to external networks and external protection layers, such as load balancers, + provide better safeguards, rate limiting is less effective. + + Therefore, we're depecating rate limiting in favor of the more reliable concurrency limiting. Gitaly RPC-based + rate limiting will be removed in GitLab 18.0. diff --git a/db/docs/ai_conversation_messages.yml b/db/docs/ai_conversation_messages.yml new file mode 100644 index 00000000000..bf361971d9f --- /dev/null +++ b/db/docs/ai_conversation_messages.yml @@ -0,0 +1,13 @@ +--- +table_name: ai_conversation_messages +classes: +- Ai::Conversation::Message +feature_categories: +- ai_abstraction_layer +- duo_chat +description: Messages for GitLab Duo features. +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/171934 +milestone: '17.7' +gitlab_schema: gitlab_main_cell +sharding_key: + organization_id: organizations diff --git a/db/docs/ai_conversation_threads.yml b/db/docs/ai_conversation_threads.yml new file mode 100644 index 00000000000..da0d247320c --- /dev/null +++ b/db/docs/ai_conversation_threads.yml @@ -0,0 +1,13 @@ +--- +table_name: ai_conversation_threads +classes: +- Ai::Conversation::Thread +feature_categories: +- ai_abstraction_layer +- duo_chat +description: Threads of messages for GitLab Duo features. +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/171934 +milestone: '17.7' +gitlab_schema: gitlab_main_cell +sharding_key: + organization_id: organizations diff --git a/db/docs/batched_background_migrations/backfill_protected_branch_merge_access_levels_protected_branch_namespace_id.yml b/db/docs/batched_background_migrations/backfill_protected_branch_merge_access_levels_protected_branch_namespace_id.yml new file mode 100644 index 00000000000..0ae07e461b4 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_protected_branch_merge_access_levels_protected_branch_namespace_id.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillProtectedBranchMergeAccessLevelsProtectedBranchNamespaceId +description: Backfills sharding key `protected_branch_merge_access_levels.protected_branch_namespace_id` from `protected_branches`. +feature_category: source_code_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/174564 +milestone: '17.7' +queued_migration_version: 20241204130230 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/batched_background_migrations/backfill_protected_branch_merge_access_levels_protected_branch_project_id.yml b/db/docs/batched_background_migrations/backfill_protected_branch_merge_access_levels_protected_branch_project_id.yml new file mode 100644 index 00000000000..bb455103c9d --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_protected_branch_merge_access_levels_protected_branch_project_id.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillProtectedBranchMergeAccessLevelsProtectedBranchProjectId +description: Backfills sharding key `protected_branch_merge_access_levels.protected_branch_project_id` from `protected_branches`. +feature_category: source_code_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/174564 +milestone: '17.7' +queued_migration_version: 20241204130225 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/batched_background_migrations/backfill_status_page_published_incidents_namespace_id.yml b/db/docs/batched_background_migrations/backfill_status_page_published_incidents_namespace_id.yml new file mode 100644 index 00000000000..6663d9b0454 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_status_page_published_incidents_namespace_id.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillStatusPagePublishedIncidentsNamespaceId +description: Backfills sharding key `status_page_published_incidents.namespace_id` from `issues`. +feature_category: incident_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/174868 +milestone: '17.7' +queued_migration_version: 20241205143060 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/protected_branch_merge_access_levels.yml b/db/docs/protected_branch_merge_access_levels.yml index 20f3e82dff6..49699a338b6 100644 --- a/db/docs/protected_branch_merge_access_levels.yml +++ b/db/docs/protected_branch_merge_access_levels.yml @@ -26,3 +26,6 @@ desired_sharding_key: sharding_key: namespace_id belongs_to: protected_branch table_size: small +desired_sharding_key_migration_job_name: +- BackfillProtectedBranchMergeAccessLevelsProtectedBranchProjectId +- BackfillProtectedBranchMergeAccessLevelsProtectedBranchNamespaceId diff --git a/db/docs/status_page_published_incidents.yml b/db/docs/status_page_published_incidents.yml index 3e8b40075a3..513a623222d 100644 --- a/db/docs/status_page_published_incidents.yml +++ b/db/docs/status_page_published_incidents.yml @@ -18,3 +18,4 @@ desired_sharding_key: sharding_key: namespace_id belongs_to: issue table_size: small +desired_sharding_key_migration_job_name: BackfillStatusPagePublishedIncidentsNamespaceId diff --git a/db/migrate/20241106183051_create_ai_conversation_threads_and_messages.rb b/db/migrate/20241106183051_create_ai_conversation_threads_and_messages.rb new file mode 100644 index 00000000000..0c3325ad065 --- /dev/null +++ b/db/migrate/20241106183051_create_ai_conversation_threads_and_messages.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class CreateAiConversationThreadsAndMessages < Gitlab::Database::Migration[2.2] + milestone '17.7' + + def change + create_table :ai_conversation_threads do |t| # rubocop:disable Migration/EnsureFactoryForTable, Lint/RedundantCopDisableDirective -- https://gitlab.com/gitlab-org/gitlab/-/issues/468630 + t.bigint :user_id, null: false + t.bigint :organization_id, null: false + t.datetime_with_timezone :last_updated_at, null: false, default: -> { 'NOW()' } + t.timestamps_with_timezone null: false + t.integer :conversation_type, limit: 2, null: false + + t.index :last_updated_at + t.index :organization_id + t.index [:user_id, :last_updated_at] + end + + create_table :ai_conversation_messages do |t| # rubocop:disable Migration/EnsureFactoryForTable, Lint/RedundantCopDisableDirective -- https://gitlab.com/gitlab-org/gitlab/-/issues/468630 + t.bigint :thread_id, null: false + t.bigint :agent_version_id, null: true + t.bigint :organization_id, null: false + t.timestamps_with_timezone null: false + t.integer :role, limit: 2, null: false + t.boolean :has_feedback, default: false + t.jsonb :extras, default: {}, null: false + t.jsonb :error_details, default: {}, null: false + t.text :content, null: false, limit: 512.kilobytes + t.text :request_xid, limit: 255 + t.text :message_xid, limit: 255 + t.text :referer_url, limit: 255 + + t.index [:thread_id, :created_at] + t.index :message_xid + t.index :organization_id + t.index :agent_version_id + end + end +end diff --git a/db/migrate/20241111143504_add_fk_to_ai_conversation_threads_and_messages.rb b/db/migrate/20241111143504_add_fk_to_ai_conversation_threads_and_messages.rb new file mode 100644 index 00000000000..30776b176a6 --- /dev/null +++ b/db/migrate/20241111143504_add_fk_to_ai_conversation_threads_and_messages.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class AddFkToAiConversationThreadsAndMessages < Gitlab::Database::Migration[2.2] + milestone '17.7' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :ai_conversation_threads, :organizations, column: :organization_id, on_delete: :cascade + add_concurrent_foreign_key :ai_conversation_threads, :users, column: :user_id, on_delete: :cascade + add_concurrent_foreign_key :ai_conversation_messages, :ai_agent_versions, column: :agent_version_id, + on_delete: :nullify + add_concurrent_foreign_key :ai_conversation_messages, :organizations, column: :organization_id, + on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :ai_conversation_threads, column: :organization_id + remove_foreign_key :ai_conversation_threads, column: :user_id + remove_foreign_key :ai_conversation_messages, column: :agent_version_id + remove_foreign_key :ai_conversation_messages, column: :organization_id + end + end +end diff --git a/db/migrate/20241127164647_track_ai_conversation_thred_record_changes.rb b/db/migrate/20241127164647_track_ai_conversation_thred_record_changes.rb new file mode 100644 index 00000000000..17ed31fde7e --- /dev/null +++ b/db/migrate/20241127164647_track_ai_conversation_thred_record_changes.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class TrackAiConversationThredRecordChanges < Gitlab::Database::Migration[2.2] + include Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers + + milestone '17.7' + + def up + track_record_deletions(:ai_conversation_threads) + end + + def down + untrack_record_deletions(:ai_conversation_threads) + end +end diff --git a/db/migrate/20241204130221_add_protected_branch_project_id_to_protected_branch_merge_access_levels.rb b/db/migrate/20241204130221_add_protected_branch_project_id_to_protected_branch_merge_access_levels.rb new file mode 100644 index 00000000000..3c8f1a43c68 --- /dev/null +++ b/db/migrate/20241204130221_add_protected_branch_project_id_to_protected_branch_merge_access_levels.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddProtectedBranchProjectIdToProtectedBranchMergeAccessLevels < Gitlab::Database::Migration[2.2] + milestone '17.7' + + def change + add_column :protected_branch_merge_access_levels, :protected_branch_project_id, :bigint + end +end diff --git a/db/migrate/20241204130226_add_protected_branch_namespace_id_to_protected_branch_merge_access_levels.rb b/db/migrate/20241204130226_add_protected_branch_namespace_id_to_protected_branch_merge_access_levels.rb new file mode 100644 index 00000000000..8c3d0320cd2 --- /dev/null +++ b/db/migrate/20241204130226_add_protected_branch_namespace_id_to_protected_branch_merge_access_levels.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddProtectedBranchNamespaceIdToProtectedBranchMergeAccessLevels < Gitlab::Database::Migration[2.2] + milestone '17.7' + + def change + add_column :protected_branch_merge_access_levels, :protected_branch_namespace_id, :bigint + end +end diff --git a/db/migrate/20241205143056_add_namespace_id_to_status_page_published_incidents.rb b/db/migrate/20241205143056_add_namespace_id_to_status_page_published_incidents.rb new file mode 100644 index 00000000000..6401a4acd56 --- /dev/null +++ b/db/migrate/20241205143056_add_namespace_id_to_status_page_published_incidents.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddNamespaceIdToStatusPagePublishedIncidents < Gitlab::Database::Migration[2.2] + milestone '17.7' + + def change + add_column :status_page_published_incidents, :namespace_id, :bigint + end +end diff --git a/db/post_migrate/20241204130222_idx_protected_branch_merge_access_levels_on_protected_branch_project_id.rb b/db/post_migrate/20241204130222_idx_protected_branch_merge_access_levels_on_protected_branch_project_id.rb new file mode 100644 index 00000000000..7a12b9a2058 --- /dev/null +++ b/db/post_migrate/20241204130222_idx_protected_branch_merge_access_levels_on_protected_branch_project_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IdxProtectedBranchMergeAccessLevelsOnProtectedBranchProjectId < Gitlab::Database::Migration[2.2] + milestone '17.7' + disable_ddl_transaction! + + INDEX_NAME = 'idx_protected_branch_merge_access_levels_protected_branch_proje' + + def up + add_concurrent_index :protected_branch_merge_access_levels, :protected_branch_project_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :protected_branch_merge_access_levels, INDEX_NAME + end +end diff --git a/db/post_migrate/20241204130223_add_protected_branch_merge_access_levels_protected_branch_project_id_fk.rb b/db/post_migrate/20241204130223_add_protected_branch_merge_access_levels_protected_branch_project_id_fk.rb new file mode 100644 index 00000000000..e762ddca308 --- /dev/null +++ b/db/post_migrate/20241204130223_add_protected_branch_merge_access_levels_protected_branch_project_id_fk.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddProtectedBranchMergeAccessLevelsProtectedBranchProjectIdFk < Gitlab::Database::Migration[2.2] + milestone '17.7' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :protected_branch_merge_access_levels, :projects, column: :protected_branch_project_id, + on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :protected_branch_merge_access_levels, column: :protected_branch_project_id + end + end +end diff --git a/db/post_migrate/20241204130224_add_protected_branch_merge_access_levels_protected_branch_project_id_trigger.rb b/db/post_migrate/20241204130224_add_protected_branch_merge_access_levels_protected_branch_project_id_trigger.rb new file mode 100644 index 00000000000..d9e1edd603e --- /dev/null +++ b/db/post_migrate/20241204130224_add_protected_branch_merge_access_levels_protected_branch_project_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddProtectedBranchMergeAccessLevelsProtectedBranchProjectIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.7' + + def up + install_sharding_key_assignment_trigger( + table: :protected_branch_merge_access_levels, + sharding_key: :protected_branch_project_id, + parent_table: :protected_branches, + parent_sharding_key: :project_id, + foreign_key: :protected_branch_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :protected_branch_merge_access_levels, + sharding_key: :protected_branch_project_id, + parent_table: :protected_branches, + parent_sharding_key: :project_id, + foreign_key: :protected_branch_id + ) + end +end diff --git a/db/post_migrate/20241204130225_queue_backfill_protected_branch_merge_access_levels_project_id.rb b/db/post_migrate/20241204130225_queue_backfill_protected_branch_merge_access_levels_project_id.rb new file mode 100644 index 00000000000..82190220171 --- /dev/null +++ b/db/post_migrate/20241204130225_queue_backfill_protected_branch_merge_access_levels_project_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillProtectedBranchMergeAccessLevelsProjectId < Gitlab::Database::Migration[2.2] + milestone '17.7' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillProtectedBranchMergeAccessLevelsProtectedBranchProjectId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :protected_branch_merge_access_levels, + :id, + :protected_branch_project_id, + :protected_branches, + :project_id, + :protected_branch_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :protected_branch_merge_access_levels, + :id, + [ + :protected_branch_project_id, + :protected_branches, + :project_id, + :protected_branch_id + ] + ) + end +end diff --git a/db/post_migrate/20241204130227_idx_protected_branch_merge_access_levels_on_protected_branch_namespace_id.rb b/db/post_migrate/20241204130227_idx_protected_branch_merge_access_levels_on_protected_branch_namespace_id.rb new file mode 100644 index 00000000000..82e91bf919e --- /dev/null +++ b/db/post_migrate/20241204130227_idx_protected_branch_merge_access_levels_on_protected_branch_namespace_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IdxProtectedBranchMergeAccessLevelsOnProtectedBranchNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.7' + disable_ddl_transaction! + + INDEX_NAME = 'idx_protected_branch_merge_access_levels_protected_branch_names' + + def up + add_concurrent_index :protected_branch_merge_access_levels, :protected_branch_namespace_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :protected_branch_merge_access_levels, INDEX_NAME + end +end diff --git a/db/post_migrate/20241204130228_add_protected_branch_merge_access_levels_protected_branch_namespace_id_fk.rb b/db/post_migrate/20241204130228_add_protected_branch_merge_access_levels_protected_branch_namespace_id_fk.rb new file mode 100644 index 00000000000..d6e0cc97584 --- /dev/null +++ b/db/post_migrate/20241204130228_add_protected_branch_merge_access_levels_protected_branch_namespace_id_fk.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddProtectedBranchMergeAccessLevelsProtectedBranchNamespaceIdFk < Gitlab::Database::Migration[2.2] + milestone '17.7' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :protected_branch_merge_access_levels, :namespaces, + column: :protected_branch_namespace_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :protected_branch_merge_access_levels, column: :protected_branch_namespace_id + end + end +end diff --git a/db/post_migrate/20241204130229_add_protected_branch_merge_access_levels_protected_branch_namespace_id_trigger.rb b/db/post_migrate/20241204130229_add_protected_branch_merge_access_levels_protected_branch_namespace_id_trigger.rb new file mode 100644 index 00000000000..209241e5f33 --- /dev/null +++ b/db/post_migrate/20241204130229_add_protected_branch_merge_access_levels_protected_branch_namespace_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddProtectedBranchMergeAccessLevelsProtectedBranchNamespaceIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.7' + + def up + install_sharding_key_assignment_trigger( + table: :protected_branch_merge_access_levels, + sharding_key: :protected_branch_namespace_id, + parent_table: :protected_branches, + parent_sharding_key: :namespace_id, + foreign_key: :protected_branch_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :protected_branch_merge_access_levels, + sharding_key: :protected_branch_namespace_id, + parent_table: :protected_branches, + parent_sharding_key: :namespace_id, + foreign_key: :protected_branch_id + ) + end +end diff --git a/db/post_migrate/20241204130230_queue_backfill_protected_branch_merge_access_levels_namespace_id.rb b/db/post_migrate/20241204130230_queue_backfill_protected_branch_merge_access_levels_namespace_id.rb new file mode 100644 index 00000000000..a3b5b0fa110 --- /dev/null +++ b/db/post_migrate/20241204130230_queue_backfill_protected_branch_merge_access_levels_namespace_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillProtectedBranchMergeAccessLevelsNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.7' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillProtectedBranchMergeAccessLevelsProtectedBranchNamespaceId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :protected_branch_merge_access_levels, + :id, + :protected_branch_namespace_id, + :protected_branches, + :namespace_id, + :protected_branch_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :protected_branch_merge_access_levels, + :id, + [ + :protected_branch_namespace_id, + :protected_branches, + :namespace_id, + :protected_branch_id + ] + ) + end +end diff --git a/db/post_migrate/20241204133318_index_project_ci_cd_settings_for_pipeline_removal.rb b/db/post_migrate/20241204133318_index_project_ci_cd_settings_for_pipeline_removal.rb new file mode 100644 index 00000000000..43686acf42d --- /dev/null +++ b/db/post_migrate/20241204133318_index_project_ci_cd_settings_for_pipeline_removal.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class IndexProjectCiCdSettingsForPipelineRemoval < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + milestone '17.7' + + OLD_INDEX = 'index_project_ci_cd_settings_on_id_partial' + INDEX = 'index_project_ci_cd_settings_on_project_id_partial' + + def up + remove_concurrent_index :project_ci_cd_settings, :id, name: OLD_INDEX + add_concurrent_index :project_ci_cd_settings, :project_id, + where: 'delete_pipelines_in_seconds IS NOT NULL', name: INDEX + end + + def down + add_concurrent_index :project_ci_cd_settings, :id, + where: 'delete_pipelines_in_seconds IS NOT NULL', name: OLD_INDEX + remove_concurrent_index :project_ci_cd_settings, :project_id, name: INDEX + end +end diff --git a/db/post_migrate/20241205143057_index_status_page_published_incidents_on_namespace_id.rb b/db/post_migrate/20241205143057_index_status_page_published_incidents_on_namespace_id.rb new file mode 100644 index 00000000000..2b114d69419 --- /dev/null +++ b/db/post_migrate/20241205143057_index_status_page_published_incidents_on_namespace_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexStatusPagePublishedIncidentsOnNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.7' + disable_ddl_transaction! + + INDEX_NAME = 'index_status_page_published_incidents_on_namespace_id' + + def up + add_concurrent_index :status_page_published_incidents, :namespace_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :status_page_published_incidents, INDEX_NAME + end +end diff --git a/db/post_migrate/20241205143058_add_status_page_published_incidents_namespace_id_fk.rb b/db/post_migrate/20241205143058_add_status_page_published_incidents_namespace_id_fk.rb new file mode 100644 index 00000000000..6367d973442 --- /dev/null +++ b/db/post_migrate/20241205143058_add_status_page_published_incidents_namespace_id_fk.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddStatusPagePublishedIncidentsNamespaceIdFk < Gitlab::Database::Migration[2.2] + milestone '17.7' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :status_page_published_incidents, :namespaces, column: :namespace_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :status_page_published_incidents, column: :namespace_id + end + end +end diff --git a/db/post_migrate/20241205143059_add_status_page_published_incidents_namespace_id_trigger.rb b/db/post_migrate/20241205143059_add_status_page_published_incidents_namespace_id_trigger.rb new file mode 100644 index 00000000000..dfc084b5006 --- /dev/null +++ b/db/post_migrate/20241205143059_add_status_page_published_incidents_namespace_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddStatusPagePublishedIncidentsNamespaceIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.7' + + def up + install_sharding_key_assignment_trigger( + table: :status_page_published_incidents, + sharding_key: :namespace_id, + parent_table: :issues, + parent_sharding_key: :namespace_id, + foreign_key: :issue_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :status_page_published_incidents, + sharding_key: :namespace_id, + parent_table: :issues, + parent_sharding_key: :namespace_id, + foreign_key: :issue_id + ) + end +end diff --git a/db/post_migrate/20241205143060_queue_backfill_status_page_published_incidents_namespace_id.rb b/db/post_migrate/20241205143060_queue_backfill_status_page_published_incidents_namespace_id.rb new file mode 100644 index 00000000000..59a0580e23f --- /dev/null +++ b/db/post_migrate/20241205143060_queue_backfill_status_page_published_incidents_namespace_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillStatusPagePublishedIncidentsNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.7' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillStatusPagePublishedIncidentsNamespaceId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :status_page_published_incidents, + :id, + :namespace_id, + :issues, + :namespace_id, + :issue_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :status_page_published_incidents, + :id, + [ + :namespace_id, + :issues, + :namespace_id, + :issue_id + ] + ) + end +end diff --git a/db/schema_migrations/20241106183051 b/db/schema_migrations/20241106183051 new file mode 100644 index 00000000000..d70712bed01 --- /dev/null +++ b/db/schema_migrations/20241106183051 @@ -0,0 +1 @@ +ab51cce35a6020a01ece32b9f7fcb9514f0298c206501669d2bff0a76247920a \ No newline at end of file diff --git a/db/schema_migrations/20241111143504 b/db/schema_migrations/20241111143504 new file mode 100644 index 00000000000..3ea6c19c03b --- /dev/null +++ b/db/schema_migrations/20241111143504 @@ -0,0 +1 @@ +3bcd882c894b0f9c60b07c4111616f8a94d2ab80cf840b2470f2d7565e9260c4 \ No newline at end of file diff --git a/db/schema_migrations/20241127164647 b/db/schema_migrations/20241127164647 new file mode 100644 index 00000000000..7efef21b973 --- /dev/null +++ b/db/schema_migrations/20241127164647 @@ -0,0 +1 @@ +c52313e55db4a50a5f3a6e8668831ffd971be7483b3cef6e8d497a500d27035d \ No newline at end of file diff --git a/db/schema_migrations/20241204130221 b/db/schema_migrations/20241204130221 new file mode 100644 index 00000000000..ad8f7d72422 --- /dev/null +++ b/db/schema_migrations/20241204130221 @@ -0,0 +1 @@ +1e016cdcb87f3794422eba3e63f9eed9ed31cadd81b60c26ae12abc5873b9559 \ No newline at end of file diff --git a/db/schema_migrations/20241204130222 b/db/schema_migrations/20241204130222 new file mode 100644 index 00000000000..1bbf5dca6e9 --- /dev/null +++ b/db/schema_migrations/20241204130222 @@ -0,0 +1 @@ +e644a608d47ef97e099df831390df81a62f65618aa8912bf811e192d723e57f9 \ No newline at end of file diff --git a/db/schema_migrations/20241204130223 b/db/schema_migrations/20241204130223 new file mode 100644 index 00000000000..c68d667a206 --- /dev/null +++ b/db/schema_migrations/20241204130223 @@ -0,0 +1 @@ +b4837be8ab67b7c0f95c9e36aab31956fa10145bf94c30e88b44caa3c1dfbda8 \ No newline at end of file diff --git a/db/schema_migrations/20241204130224 b/db/schema_migrations/20241204130224 new file mode 100644 index 00000000000..334901f1b43 --- /dev/null +++ b/db/schema_migrations/20241204130224 @@ -0,0 +1 @@ +5d9bf71d27193ebb27d098439581721a3b84e454dee0d1be3d47b1d3bec03694 \ No newline at end of file diff --git a/db/schema_migrations/20241204130225 b/db/schema_migrations/20241204130225 new file mode 100644 index 00000000000..5c51eb21082 --- /dev/null +++ b/db/schema_migrations/20241204130225 @@ -0,0 +1 @@ +f19df9f4978799b7eefaa9a5eb16d5add079ae7d6bfb34b808307cfcf06811d0 \ No newline at end of file diff --git a/db/schema_migrations/20241204130226 b/db/schema_migrations/20241204130226 new file mode 100644 index 00000000000..88b49e2ad6e --- /dev/null +++ b/db/schema_migrations/20241204130226 @@ -0,0 +1 @@ +d65257f8c1c805654fb1da7f6cfe54504c3c98e5ae0b24c18cd11749eb8ea1d4 \ No newline at end of file diff --git a/db/schema_migrations/20241204130227 b/db/schema_migrations/20241204130227 new file mode 100644 index 00000000000..caf83882b2c --- /dev/null +++ b/db/schema_migrations/20241204130227 @@ -0,0 +1 @@ +2d00490a000507a683dcd7f43de9440a2f1800996aeaa32955d1a8df2262da15 \ No newline at end of file diff --git a/db/schema_migrations/20241204130228 b/db/schema_migrations/20241204130228 new file mode 100644 index 00000000000..e41e0dac1ee --- /dev/null +++ b/db/schema_migrations/20241204130228 @@ -0,0 +1 @@ +feeb2affc1884ea5eabb904f6788da6c6407233868b9bf13f9cb870d4ab33a61 \ No newline at end of file diff --git a/db/schema_migrations/20241204130229 b/db/schema_migrations/20241204130229 new file mode 100644 index 00000000000..1a8254a03be --- /dev/null +++ b/db/schema_migrations/20241204130229 @@ -0,0 +1 @@ +b6826a4f6f4383b28d1990275ae8243425efac585c4bc0883ab0477d5068e8cc \ No newline at end of file diff --git a/db/schema_migrations/20241204130230 b/db/schema_migrations/20241204130230 new file mode 100644 index 00000000000..8ec34197b6f --- /dev/null +++ b/db/schema_migrations/20241204130230 @@ -0,0 +1 @@ +f2301d9ebd4aa0e8f18f32b9c467031fd9872e58a5868bb613b883d2019e670a \ No newline at end of file diff --git a/db/schema_migrations/20241204133318 b/db/schema_migrations/20241204133318 new file mode 100644 index 00000000000..e572d377fc8 --- /dev/null +++ b/db/schema_migrations/20241204133318 @@ -0,0 +1 @@ +d1e67db49bbfb3c65aa8d0bff2de46bd78dc3057e2338344c33f684ddfb6a06c \ No newline at end of file diff --git a/db/schema_migrations/20241205143056 b/db/schema_migrations/20241205143056 new file mode 100644 index 00000000000..5a30f8c419f --- /dev/null +++ b/db/schema_migrations/20241205143056 @@ -0,0 +1 @@ +be2cd9f5c5cf4fa9ba941735ace989c6b4a16b50150ec6e5fac4125653a947ce \ No newline at end of file diff --git a/db/schema_migrations/20241205143057 b/db/schema_migrations/20241205143057 new file mode 100644 index 00000000000..f4975fa0b38 --- /dev/null +++ b/db/schema_migrations/20241205143057 @@ -0,0 +1 @@ +d900f3896f99f8bf65c776cc5c6d4d20cb4bf9e20ab5651df4fc5b9f8fccc843 \ No newline at end of file diff --git a/db/schema_migrations/20241205143058 b/db/schema_migrations/20241205143058 new file mode 100644 index 00000000000..72895972840 --- /dev/null +++ b/db/schema_migrations/20241205143058 @@ -0,0 +1 @@ +62778787a1e3bb41291887c93a8aec7fd60bfadb808ad2d1b4283fb5ab227fca \ No newline at end of file diff --git a/db/schema_migrations/20241205143059 b/db/schema_migrations/20241205143059 new file mode 100644 index 00000000000..32dbdbf98b7 --- /dev/null +++ b/db/schema_migrations/20241205143059 @@ -0,0 +1 @@ +88e34456142cf48b55d79a59c88934387eb98d7ffa97a0450656dc54ed3dcd1b \ No newline at end of file diff --git a/db/schema_migrations/20241205143060 b/db/schema_migrations/20241205143060 new file mode 100644 index 00000000000..85da45db6a4 --- /dev/null +++ b/db/schema_migrations/20241205143060 @@ -0,0 +1 @@ +fe926f2a168130492492e670268c9e32ce860150db85ceffae37f4b0f58ea9f6 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 7d9c2146569..fd329190b34 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -978,6 +978,22 @@ RETURN NEW; END $$; +CREATE FUNCTION trigger_05cc4448a8aa() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."protected_branch_namespace_id" IS NULL THEN + SELECT "namespace_id" + INTO NEW."protected_branch_namespace_id" + FROM "protected_branches" + WHERE "protected_branches"."id" = NEW."protected_branch_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_05ce163deddf() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -1026,6 +1042,22 @@ RETURN NEW; END $$; +CREATE FUNCTION trigger_0aea02e5a699() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."protected_branch_project_id" IS NULL THEN + SELECT "project_id" + INTO NEW."protected_branch_project_id" + FROM "protected_branches" + WHERE "protected_branches"."id" = NEW."protected_branch_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_0c326daf67cf() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -2865,6 +2897,22 @@ RETURN NEW; END $$; +CREATE FUNCTION trigger_dbe374a57cbb() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."namespace_id" IS NULL THEN + SELECT "namespace_id" + INTO NEW."namespace_id" + FROM "issues" + WHERE "issues"."id" = NEW."issue_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_dc13168b8025() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -6103,6 +6151,55 @@ CREATE SEQUENCE ai_code_suggestion_events_id_seq ALTER SEQUENCE ai_code_suggestion_events_id_seq OWNED BY ai_code_suggestion_events.id; +CREATE TABLE ai_conversation_messages ( + id bigint NOT NULL, + thread_id bigint NOT NULL, + agent_version_id bigint, + organization_id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + role smallint NOT NULL, + has_feedback boolean DEFAULT false, + extras jsonb DEFAULT '{}'::jsonb NOT NULL, + error_details jsonb DEFAULT '{}'::jsonb NOT NULL, + content text NOT NULL, + request_xid text, + message_xid text, + referer_url text, + CONSTRAINT check_0fe78937e4 CHECK ((char_length(content) <= 524288)), + CONSTRAINT check_8daec62ec9 CHECK ((char_length(request_xid) <= 255)), + CONSTRAINT check_b14b137e02 CHECK ((char_length(message_xid) <= 255)), + CONSTRAINT check_f36c73d1d9 CHECK ((char_length(referer_url) <= 255)) +); + +CREATE SEQUENCE ai_conversation_messages_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE ai_conversation_messages_id_seq OWNED BY ai_conversation_messages.id; + +CREATE TABLE ai_conversation_threads ( + id bigint NOT NULL, + user_id bigint NOT NULL, + organization_id bigint NOT NULL, + last_updated_at timestamp with time zone DEFAULT now() NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + conversation_type smallint NOT NULL +); + +CREATE SEQUENCE ai_conversation_threads_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE ai_conversation_threads_id_seq OWNED BY ai_conversation_threads.id; + CREATE TABLE ai_feature_settings ( id bigint NOT NULL, created_at timestamp with time zone NOT NULL, @@ -18619,7 +18716,9 @@ CREATE TABLE protected_branch_merge_access_levels ( created_at timestamp without time zone NOT NULL, updated_at timestamp without time zone NOT NULL, user_id bigint, - group_id bigint + group_id bigint, + protected_branch_project_id bigint, + protected_branch_namespace_id bigint ); CREATE SEQUENCE protected_branch_merge_access_levels_id_seq @@ -20182,7 +20281,8 @@ CREATE TABLE status_page_published_incidents ( id bigint NOT NULL, created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, - issue_id bigint NOT NULL + issue_id bigint NOT NULL, + namespace_id bigint ); CREATE SEQUENCE status_page_published_incidents_id_seq @@ -23139,6 +23239,10 @@ ALTER TABLE ONLY ai_agents ALTER COLUMN id SET DEFAULT nextval('ai_agents_id_seq ALTER TABLE ONLY ai_code_suggestion_events ALTER COLUMN id SET DEFAULT nextval('ai_code_suggestion_events_id_seq'::regclass); +ALTER TABLE ONLY ai_conversation_messages ALTER COLUMN id SET DEFAULT nextval('ai_conversation_messages_id_seq'::regclass); + +ALTER TABLE ONLY ai_conversation_threads ALTER COLUMN id SET DEFAULT nextval('ai_conversation_threads_id_seq'::regclass); + ALTER TABLE ONLY ai_feature_settings ALTER COLUMN id SET DEFAULT nextval('ai_feature_settings_id_seq'::regclass); ALTER TABLE ONLY ai_self_hosted_models ALTER COLUMN id SET DEFAULT nextval('ai_self_hosted_models_id_seq'::regclass); @@ -24969,6 +25073,12 @@ ALTER TABLE ONLY ai_agents ALTER TABLE ONLY ai_code_suggestion_events ADD CONSTRAINT ai_code_suggestion_events_pkey PRIMARY KEY (id, "timestamp"); +ALTER TABLE ONLY ai_conversation_messages + ADD CONSTRAINT ai_conversation_messages_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY ai_conversation_threads + ADD CONSTRAINT ai_conversation_threads_pkey PRIMARY KEY (id); + ALTER TABLE ONLY ai_feature_settings ADD CONSTRAINT ai_feature_settings_pkey PRIMARY KEY (id); @@ -28961,6 +29071,10 @@ CREATE INDEX idx_projects_id_created_at_disable_overriding_approvers_true ON pro CREATE INDEX idx_projects_on_repository_storage_last_repository_updated_at ON projects USING btree (id, repository_storage, last_repository_updated_at); +CREATE INDEX idx_protected_branch_merge_access_levels_protected_branch_names ON protected_branch_merge_access_levels USING btree (protected_branch_namespace_id); + +CREATE INDEX idx_protected_branch_merge_access_levels_protected_branch_proje ON protected_branch_merge_access_levels USING btree (protected_branch_project_id); + CREATE INDEX idx_reminder_frequency_on_work_item_progresses ON work_item_progresses USING btree (reminder_frequency); CREATE INDEX idx_sbom_components_on_name_gin ON sbom_components USING gin (name gin_trgm_ops); @@ -29171,6 +29285,20 @@ CREATE INDEX index_ai_code_suggestion_events_on_organization_id ON ONLY ai_code_ CREATE INDEX index_ai_code_suggestion_events_on_user_id ON ONLY ai_code_suggestion_events USING btree (user_id); +CREATE INDEX index_ai_conversation_messages_on_agent_version_id ON ai_conversation_messages USING btree (agent_version_id); + +CREATE INDEX index_ai_conversation_messages_on_message_xid ON ai_conversation_messages USING btree (message_xid); + +CREATE INDEX index_ai_conversation_messages_on_organization_id ON ai_conversation_messages USING btree (organization_id); + +CREATE INDEX index_ai_conversation_messages_on_thread_id_and_created_at ON ai_conversation_messages USING btree (thread_id, created_at); + +CREATE INDEX index_ai_conversation_threads_on_last_updated_at ON ai_conversation_threads USING btree (last_updated_at); + +CREATE INDEX index_ai_conversation_threads_on_organization_id ON ai_conversation_threads USING btree (organization_id); + +CREATE INDEX index_ai_conversation_threads_on_user_id_and_last_updated_at ON ai_conversation_threads USING btree (user_id, last_updated_at); + CREATE INDEX index_ai_feature_settings_on_ai_self_hosted_model_id ON ai_feature_settings USING btree (ai_self_hosted_model_id); CREATE UNIQUE INDEX index_ai_feature_settings_on_feature ON ai_feature_settings USING btree (feature); @@ -31981,10 +32109,10 @@ CREATE UNIQUE INDEX index_project_auto_devops_on_project_id ON project_auto_devo CREATE UNIQUE INDEX index_project_build_artifacts_size_refreshes_on_project_id ON project_build_artifacts_size_refreshes USING btree (project_id); -CREATE INDEX index_project_ci_cd_settings_on_id_partial ON project_ci_cd_settings USING btree (id) WHERE (delete_pipelines_in_seconds IS NOT NULL); - CREATE UNIQUE INDEX index_project_ci_cd_settings_on_project_id ON project_ci_cd_settings USING btree (project_id); +CREATE INDEX index_project_ci_cd_settings_on_project_id_partial ON project_ci_cd_settings USING btree (project_id) WHERE (delete_pipelines_in_seconds IS NOT NULL); + CREATE UNIQUE INDEX index_project_ci_feature_usages_unique_columns ON project_ci_feature_usages USING btree (project_id, feature, default_branch); CREATE INDEX index_project_compliance_framework_settings_on_framework_id ON project_compliance_framework_settings USING btree (framework_id); @@ -32659,6 +32787,8 @@ CREATE INDEX index_status_check_responses_on_project_id ON status_check_response CREATE UNIQUE INDEX index_status_page_published_incidents_on_issue_id ON status_page_published_incidents USING btree (issue_id); +CREATE INDEX index_status_page_published_incidents_on_namespace_id ON status_page_published_incidents USING btree (namespace_id); + CREATE INDEX index_status_page_settings_on_project_id ON status_page_settings USING btree (project_id); CREATE INDEX index_subscription_add_on_purchases_on_namespace_id_add_on_id ON subscription_add_on_purchases USING btree (namespace_id, subscription_add_on_id); @@ -35443,6 +35573,8 @@ ALTER INDEX p_ci_job_artifacts_expire_at_job_id_idx1 ATTACH PARTITION tmp_index_ ALTER INDEX p_ci_builds_token_encrypted_partition_id_idx ATTACH PARTITION unique_ci_builds_token_encrypted_and_partition_id; +CREATE TRIGGER ai_conversation_threads_loose_fk_trigger AFTER DELETE ON ai_conversation_threads REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records(); + CREATE TRIGGER assign_ci_runner_taggings_id_trigger BEFORE INSERT ON ci_runner_taggings FOR EACH ROW EXECUTE FUNCTION assign_ci_runner_taggings_id_value(); CREATE TRIGGER assign_p_ci_build_tags_id_trigger BEFORE INSERT ON p_ci_build_tags FOR EACH ROW EXECUTE FUNCTION assign_p_ci_build_tags_id_value(); @@ -35517,12 +35649,16 @@ CREATE TRIGGER trigger_02450faab875 BEFORE INSERT OR UPDATE ON vulnerability_occ CREATE TRIGGER trigger_038fe84feff7 BEFORE INSERT OR UPDATE ON approvals FOR EACH ROW EXECUTE FUNCTION trigger_038fe84feff7(); +CREATE TRIGGER trigger_05cc4448a8aa BEFORE INSERT OR UPDATE ON protected_branch_merge_access_levels FOR EACH ROW EXECUTE FUNCTION trigger_05cc4448a8aa(); + CREATE TRIGGER trigger_05ce163deddf BEFORE INSERT OR UPDATE ON status_check_responses FOR EACH ROW EXECUTE FUNCTION trigger_05ce163deddf(); CREATE TRIGGER trigger_0a1b0adcf686 BEFORE INSERT OR UPDATE ON packages_debian_project_components FOR EACH ROW EXECUTE FUNCTION trigger_0a1b0adcf686(); CREATE TRIGGER trigger_0a29d4d42b62 BEFORE INSERT OR UPDATE ON approval_project_rules_protected_branches FOR EACH ROW EXECUTE FUNCTION trigger_0a29d4d42b62(); +CREATE TRIGGER trigger_0aea02e5a699 BEFORE INSERT OR UPDATE ON protected_branch_merge_access_levels FOR EACH ROW EXECUTE FUNCTION trigger_0aea02e5a699(); + CREATE TRIGGER trigger_0c326daf67cf BEFORE INSERT OR UPDATE ON analytics_cycle_analytics_value_stream_settings FOR EACH ROW EXECUTE FUNCTION trigger_0c326daf67cf(); CREATE TRIGGER trigger_0da002390fdc BEFORE INSERT OR UPDATE ON operations_feature_flags_issues FOR EACH ROW EXECUTE FUNCTION trigger_0da002390fdc(); @@ -35755,6 +35891,8 @@ CREATE TRIGGER trigger_dadd660afe2c BEFORE INSERT OR UPDATE ON packages_debian_g CREATE TRIGGER trigger_dbdd61a66a91 BEFORE INSERT OR UPDATE ON agent_activity_events FOR EACH ROW EXECUTE FUNCTION trigger_dbdd61a66a91(); +CREATE TRIGGER trigger_dbe374a57cbb BEFORE INSERT OR UPDATE ON status_page_published_incidents FOR EACH ROW EXECUTE FUNCTION trigger_dbe374a57cbb(); + CREATE TRIGGER trigger_dc13168b8025 BEFORE INSERT OR UPDATE ON vulnerability_flags FOR EACH ROW EXECUTE FUNCTION trigger_dc13168b8025(); CREATE TRIGGER trigger_delete_project_namespace_on_project_delete AFTER DELETE ON projects FOR EACH ROW WHEN ((old.project_namespace_id IS NOT NULL)) EXECUTE FUNCTION delete_associated_project_namespace(); @@ -35843,6 +35981,9 @@ CREATE TRIGGER users_loose_fk_trigger AFTER DELETE ON users REFERENCING OLD TABL CREATE TRIGGER virtual_registries_packages_maven_upstreams_loose_fk_trigger AFTER DELETE ON virtual_registries_packages_maven_upstreams REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records(); +ALTER TABLE ONLY ai_conversation_threads + ADD CONSTRAINT fk_00234c7444 FOREIGN KEY (organization_id) REFERENCES organizations(id) ON DELETE CASCADE; + ALTER TABLE ONLY deployments ADD CONSTRAINT fk_009fd21147 FOREIGN KEY (environment_id) REFERENCES environments(id) ON DELETE CASCADE; @@ -36221,6 +36362,9 @@ ALTER TABLE ONLY operations_feature_flags_issues ALTER TABLE ONLY push_event_payloads ADD CONSTRAINT fk_36c74129da FOREIGN KEY (event_id) REFERENCES events(id) ON DELETE CASCADE; +ALTER TABLE ONLY protected_branch_merge_access_levels + ADD CONSTRAINT fk_37ab3dd3ba FOREIGN KEY (protected_branch_project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY protected_tag_create_access_levels ADD CONSTRAINT fk_386a642e13 FOREIGN KEY (deploy_key_id) REFERENCES keys(id) ON DELETE CASCADE; @@ -36515,6 +36659,9 @@ ALTER TABLE ONLY import_placeholder_memberships ALTER TABLE p_ci_builds ADD CONSTRAINT fk_6661f4f0e8 FOREIGN KEY (resource_group_id) REFERENCES ci_resource_groups(id) ON DELETE SET NULL; +ALTER TABLE ONLY ai_conversation_messages + ADD CONSTRAINT fk_68774ec148 FOREIGN KEY (organization_id) REFERENCES organizations(id) ON DELETE CASCADE; + ALTER TABLE ONLY remote_development_agent_configs ADD CONSTRAINT fk_6a09894a0f FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; @@ -36941,6 +37088,9 @@ ALTER TABLE ONLY merge_requests_closing_issues ALTER TABLE ONLY issue_assignment_events ADD CONSTRAINT fk_a989e2acd0 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; +ALTER TABLE ONLY status_page_published_incidents + ADD CONSTRAINT fk_a9fb727793 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY ssh_signatures ADD CONSTRAINT fk_aa1efbe865 FOREIGN KEY (key_id) REFERENCES keys(id) ON DELETE SET NULL; @@ -37025,6 +37175,9 @@ ALTER TABLE ONLY status_check_responses ALTER TABLE ONLY packages_dependency_links ADD CONSTRAINT fk_b5c56b6ede FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; +ALTER TABLE ONLY ai_conversation_messages + ADD CONSTRAINT fk_b5d715b1e4 FOREIGN KEY (agent_version_id) REFERENCES ai_agent_versions(id) ON DELETE SET NULL; + ALTER TABLE ONLY compliance_framework_security_policies ADD CONSTRAINT fk_b5df066d8f FOREIGN KEY (framework_id) REFERENCES compliance_management_frameworks(id) ON DELETE CASCADE; @@ -37307,6 +37460,9 @@ ALTER TABLE ONLY sbom_occurrences ALTER TABLE ONLY todos ADD CONSTRAINT fk_d94154aa95 FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; +ALTER TABLE ONLY ai_conversation_threads + ADD CONSTRAINT fk_d97014a270 FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; + ALTER TABLE ONLY label_links ADD CONSTRAINT fk_d97dd08678 FOREIGN KEY (label_id) REFERENCES labels(id) ON DELETE CASCADE; @@ -37508,6 +37664,9 @@ ALTER TABLE ONLY scan_result_policy_violations ALTER TABLE ONLY analytics_devops_adoption_segments ADD CONSTRAINT fk_f5aa768998 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; +ALTER TABLE ONLY protected_branch_merge_access_levels + ADD CONSTRAINT fk_f5acff2bb8 FOREIGN KEY (protected_branch_namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY boards_epic_list_user_preferences ADD CONSTRAINT fk_f5f2fe5c1f FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; diff --git a/doc/administration/gitaly/monitoring.md b/doc/administration/gitaly/monitoring.md index a4dc13f7841..06fb53a02be 100644 --- a/doc/administration/gitaly/monitoring.md +++ b/doc/administration/gitaly/monitoring.md @@ -15,7 +15,13 @@ Metric definitions are available: - Using [Grafana Explore](https://grafana.com/docs/grafana/latest/explore/) on a Grafana instance configured against Prometheus. -## Monitor Gitaly rate limiting + + +## Monitor Gitaly rate limiting (deprecated) + +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitaly/-/issues/5011) in GitLab 17.7 +and is planned for removal in 18.0. Use [concurrency limiting](concurrency_limiting.md) instead. Gitaly can be configured to limit requests based on: @@ -29,6 +35,8 @@ of requests dropped due to request limiting. The `reason` label indicates why a - `max_size`, because the concurrency queue size was reached. - `max_time`, because the request exceeded the maximum queue wait time as configured in Gitaly. + + ## Monitor Gitaly concurrency limiting You can observe specific behavior of [concurrency-queued requests](concurrency_limiting.md#limit-rpc-concurrency) using Gitaly logs and Prometheus. diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index a08f604a763..8b3bfbafded 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -18063,6 +18063,7 @@ A user with add-on data. | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | | `twitter` | [`String`](#string) | X (formerly Twitter) username of the user. | +| `type` | [`UserType!`](#usertype) | Type of the user. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `userPreferences` | [`UserPreferences`](#userpreferences) | Preferences for the user. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | @@ -19019,6 +19020,7 @@ Core representation of a GitLab user. | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | | `twitter` | [`String`](#string) | X (formerly Twitter) username of the user. | +| `type` | [`UserType!`](#usertype) | Type of the user. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `userPreferences` | [`UserPreferences`](#userpreferences) | Preferences for the user. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | @@ -21571,6 +21573,7 @@ The currently authenticated GitLab user. | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | | `twitter` | [`String`](#string) | X (formerly Twitter) username of the user. | +| `type` | [`UserType!`](#usertype) | Type of the user. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `userPreferences` | [`UserPreferences`](#userpreferences) | Preferences for the user. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | @@ -27530,6 +27533,7 @@ A user assigned to a merge request. | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | | `twitter` | [`String`](#string) | X (formerly Twitter) username of the user. | +| `type` | [`UserType!`](#usertype) | Type of the user. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `userPreferences` | [`UserPreferences`](#userpreferences) | Preferences for the user. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | @@ -27935,6 +27939,7 @@ The author of the merge request. | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | | `twitter` | [`String`](#string) | X (formerly Twitter) username of the user. | +| `type` | [`UserType!`](#usertype) | Type of the user. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `userPreferences` | [`UserPreferences`](#userpreferences) | Preferences for the user. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | @@ -28386,6 +28391,7 @@ A user participating in a merge request. | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | | `twitter` | [`String`](#string) | X (formerly Twitter) username of the user. | +| `type` | [`UserType!`](#usertype) | Type of the user. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `userPreferences` | [`UserPreferences`](#userpreferences) | Preferences for the user. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | @@ -28810,6 +28816,7 @@ A user assigned to a merge request as a reviewer. | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | | `twitter` | [`String`](#string) | X (formerly Twitter) username of the user. | +| `type` | [`UserType!`](#usertype) | Type of the user. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `userPreferences` | [`UserPreferences`](#userpreferences) | Preferences for the user. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | @@ -35600,6 +35607,7 @@ Core representation of a GitLab user. | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | | `twitter` | [`String`](#string) | X (formerly Twitter) username of the user. | +| `type` | [`UserType!`](#usertype) | Type of the user. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `userPreferences` | [`UserPreferences`](#userpreferences) | Preferences for the user. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | @@ -37741,6 +37749,7 @@ LLMs supported by the self-hosted model features. | `GPT` | GPT: Suitable for code suggestions. | | `LLAMA3` | LLaMA 3: Suitable for code suggestions and duo chat. | | `MISTRAL` | Mistral: Suitable for code suggestions and duo chat. | +| `MIXTRAL` | Mixtral: Suitable for code suggestions and duo chat. | ### `AiAction` @@ -40835,6 +40844,31 @@ Possible states of a user. | `deactivated` | User is no longer active and cannot use the system. | | `ldap_blocked` | User has been blocked by the system. | +### `UserType` + +Possible types of user. + +| Value | Description | +| ----- | ----------- | +| `ADMIN_BOT` | Admin bot. | +| `ALERT_BOT` | Alert bot. | +| `AUTOMATION_BOT` | Automation bot. | +| `DUO_CODE_REVIEW_BOT` | Duo code review bot. | +| `GHOST` | Ghost. | +| `HUMAN` | Human. | +| `IMPORT_USER` | Import user. | +| `LLM_BOT` | Llm bot. | +| `MIGRATION_BOT` | Migration bot. | +| `PLACEHOLDER` | Placeholder. | +| `PROJECT_BOT` | Project bot. | +| `SECURITY_BOT` | Security bot. | +| `SECURITY_POLICY_BOT` | Security policy bot. | +| `SERVICE_ACCOUNT` | Service account. | +| `SERVICE_USER` | Service user. | +| `SUGGESTED_REVIEWERS_BOT` | Suggested reviewers bot. | +| `SUPPORT_BOT` | Support bot. | +| `VISUAL_REVIEW_BOT` | Visual review bot. | + ### `ValueStreamDashboardMetric` Possible identifier types for a measurement. @@ -43127,6 +43161,7 @@ Implementations: | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | | `twitter` | [`String`](#string) | X (formerly Twitter) username of the user. | +| `type` | [`UserType!`](#usertype) | Type of the user. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `userPreferences` | [`UserPreferences`](#userpreferences) | Preferences for the user. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | diff --git a/doc/ci/environments/protected_environments.md b/doc/ci/environments/protected_environments.md index f3e594e3756..c9e92fae533 100644 --- a/doc/ci/environments/protected_environments.md +++ b/doc/ci/environments/protected_environments.md @@ -171,7 +171,11 @@ Maintainers can: - Unprotect a protected environment by selecting the **Unprotect** button for that environment. After an environment is unprotected, all access entries are deleted and must -be re-entered if the environment is re-protected. +be re-entered if the environment is re-protected. + +After an approval rule is deleted, previously approved deployments do not show who approved the deployment. +Information on who approved a deployment is still available in the [project audit events](../../user/compliance/audit_events.md#project-audit-events). +If a new rule is added, previous deployments show the new rules without the option to approve the deployment. [Issue 506687](https://gitlab.com/gitlab-org/gitlab/-/issues/506687) proposes to show the full approval history of deployments, even if an approval rule is deleted. For more information, see [Deployment safety](deployment_safety.md). diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md index b5277e29f3c..637b4fbfc70 100644 --- a/doc/update/deprecations.md +++ b/doc/update/deprecations.md @@ -562,6 +562,35 @@ You can read more about it in the [charts release page](https://docs.gitlab.com/
+### Gitaly rate limiting + +
+ +- Announced in GitLab 17.7 +- Removal in GitLab 18.0 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitaly/-/issues/5011). + +
+ +Because of the highly variable nature of Git operations and repository latencies, Gitaly +[RPC-based rate limiting](https://docs.gitlab.com/ee/administration/gitaly/monitoring.html#monitor-gitaly-rate-limiting) +is ineffective. Configuring proper rate limits is challenging and often becomes obsolete quickly because harmful +actions rarely generate enough requests per second to stand out. + +Gitaly already supports [concurrency limiting](https://docs.gitlab.com/ee/administration/gitaly/concurrency_limiting.html) and an +[adaptive limiting add-on](https://docs.gitlab.com/ee/administration/gitaly/concurrency_limiting.html#adaptive-concurrency-limiting), +which have proven to work well in production. + +Because Gitaly is not directly exposed to external networks and external protection layers, such as load balancers, +provide better safeguards, rate limiting is less effective. + +Therefore, we're depecating rate limiting in favor of the more reliable concurrency limiting. Gitaly RPC-based +rate limiting will be removed in GitLab 18.0. + +
+ +
+ ### Group vulnerability report by OWASP top 10 2017 is deprecated
diff --git a/doc/user/application_security/policies/scan_execution_policies.md b/doc/user/application_security/policies/scan_execution_policies.md index 08c28ccddf6..a5d90448520 100644 --- a/doc/user/application_security/policies/scan_execution_policies.md +++ b/doc/user/application_security/policies/scan_execution_policies.md @@ -140,6 +140,7 @@ This rule enforces the defined actions whenever the pipeline runs for a selected > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/152855) a new application setting `security_policy_scheduled_scans_max_concurrency` in GitLab 17.1. The concurrency limit applies when both the `scan_execution_pipeline_worker` and `scan_execution_pipeline_concurrency_control` are enabled. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/158636) a concurrency limit for scan execution scheduled jobs in GitLab 17.3 [with a flag](../../../administration/feature_flags.md) named `scan_execution_pipeline_concurrency_control`. > - [Enabled](https://gitlab.com/gitlab-org/gitlab/-/issues/451890) the `scan_execution_pipeline_worker` feature flag on GitLab.com in GitLab 17.5. +> - [Feature flag](https://gitlab.com/gitlab-org/gitlab/-/issues/451890) `scan_execution_pipeline_worker` removed in GitLab 17.6. > - [Enabled](https://gitlab.com/gitlab-org/gitlab/-/issues/463802) the `scan_execution_pipeline_concurrency_control` feature flag on GitLab.com in GitLab 17.6. WARNING: diff --git a/lib/gitlab/background_migration/backfill_protected_branch_merge_access_levels_protected_branch_namespace_id.rb b/lib/gitlab/background_migration/backfill_protected_branch_merge_access_levels_protected_branch_namespace_id.rb new file mode 100644 index 00000000000..a06d59345bd --- /dev/null +++ b/lib/gitlab/background_migration/backfill_protected_branch_merge_access_levels_protected_branch_namespace_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillProtectedBranchMergeAccessLevelsProtectedBranchNamespaceId < BackfillDesiredShardingKeyJob + operation_name :backfill_protected_branch_merge_access_levels_protected_branch_namespace_id + feature_category :source_code_management + end + end +end diff --git a/lib/gitlab/background_migration/backfill_protected_branch_merge_access_levels_protected_branch_project_id.rb b/lib/gitlab/background_migration/backfill_protected_branch_merge_access_levels_protected_branch_project_id.rb new file mode 100644 index 00000000000..1a8d6f584cf --- /dev/null +++ b/lib/gitlab/background_migration/backfill_protected_branch_merge_access_levels_protected_branch_project_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillProtectedBranchMergeAccessLevelsProtectedBranchProjectId < BackfillDesiredShardingKeyJob + operation_name :backfill_protected_branch_merge_access_levels_protected_branch_project_id + feature_category :source_code_management + end + end +end diff --git a/lib/gitlab/background_migration/backfill_status_page_published_incidents_namespace_id.rb b/lib/gitlab/background_migration/backfill_status_page_published_incidents_namespace_id.rb new file mode 100644 index 00000000000..f2046fe7a59 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_status_page_published_incidents_namespace_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillStatusPagePublishedIncidentsNamespaceId < BackfillDesiredShardingKeyJob + operation_name :backfill_status_page_published_incidents_namespace_id + feature_category :incident_management + end + end +end diff --git a/lib/gitlab/ci/tags/bulk_insert.rb b/lib/gitlab/ci/tags/bulk_insert.rb index 70eb3200026..2f8831f9354 100644 --- a/lib/gitlab/ci/tags/bulk_insert.rb +++ b/lib/gitlab/ci/tags/bulk_insert.rb @@ -95,7 +95,7 @@ module Gitlab tags.each do |tag| accumulator[:taggings] << tagging_attributes(tag, taggable) if polymorphic_taggings_available? - if monomorphic_taggings_available? + if monomorphic_taggings_available?(taggable) accumulator[:monomorphic_taggings] << monomorphic_taggings_record(tag, taggable) end end @@ -129,8 +129,8 @@ module Gitlab end end - def monomorphic_taggings_available? - config.monomorphic_taggings? + def monomorphic_taggings_available?(taggable) + config.monomorphic_taggings?(taggable) end def polymorphic_taggings_available? diff --git a/lib/gitlab/ci/tags/bulk_insert/builds_tags_configuration.rb b/lib/gitlab/ci/tags/bulk_insert/builds_tags_configuration.rb index 139d0505184..c6dc8066033 100644 --- a/lib/gitlab/ci/tags/bulk_insert/builds_tags_configuration.rb +++ b/lib/gitlab/ci/tags/bulk_insert/builds_tags_configuration.rb @@ -37,7 +37,7 @@ module Gitlab true end - def monomorphic_taggings? + def monomorphic_taggings?(_taggable) true end end diff --git a/lib/gitlab/ci/tags/bulk_insert/configuration_factory.rb b/lib/gitlab/ci/tags/bulk_insert/configuration_factory.rb index 3c49730131c..9dff9d15e35 100644 --- a/lib/gitlab/ci/tags/bulk_insert/configuration_factory.rb +++ b/lib/gitlab/ci/tags/bulk_insert/configuration_factory.rb @@ -23,7 +23,8 @@ module Gitlab def strategies [ - BuildsTagsConfiguration + BuildsTagsConfiguration, + RunnerTaggingsConfiguration ] end end diff --git a/lib/gitlab/ci/tags/bulk_insert/no_config.rb b/lib/gitlab/ci/tags/bulk_insert/no_config.rb index cb568451360..57b2aadeb0a 100644 --- a/lib/gitlab/ci/tags/bulk_insert/no_config.rb +++ b/lib/gitlab/ci/tags/bulk_insert/no_config.rb @@ -15,7 +15,7 @@ module Gitlab true end - def monomorphic_taggings? + def monomorphic_taggings?(_taggable) false end end diff --git a/lib/gitlab/ci/tags/bulk_insert/runner_taggings_configuration.rb b/lib/gitlab/ci/tags/bulk_insert/runner_taggings_configuration.rb new file mode 100644 index 00000000000..b8dabba521b --- /dev/null +++ b/lib/gitlab/ci/tags/bulk_insert/runner_taggings_configuration.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Tags + class BulkInsert + class RunnerTaggingsConfiguration + include ::Gitlab::Utils::StrongMemoize + + def self.applies_to?(record) + record.is_a?(::Ci::Runner) + end + + def self.build_from(runner) + new(runner) + end + + def initialize(runner) + @runner = runner + end + + def join_model + ::Ci::RunnerTagging + end + + def unique_by + [:tag_id, :runner_id, :runner_type] + end + + def attributes_map(runner) + { + runner_id: runner.id, + runner_type: runner.runner_type, + sharding_key_id: runner.sharding_key_id + } + end + + def polymorphic_taggings? + true + end + + def monomorphic_taggings?(runner) + strong_memoize_with(:monomorphic_taggings, runner.owner) do + ::Feature.enabled?(:write_to_ci_runner_taggings, runner.owner) + end + end + end + end + end + end +end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index f8aaaff55d8..dae95ad1edb 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -25285,6 +25285,12 @@ msgstr "" msgid "GlobalSearch|Archived" msgstr "" +msgid "GlobalSearch|Author" +msgstr "" + +msgid "GlobalSearch|Author not included" +msgstr "" + msgid "GlobalSearch|Branch not included" msgstr "" @@ -48297,7 +48303,7 @@ msgstr "" msgid "ScanExecutionPolicy|Add new condition" msgstr "" -msgid "ScanExecutionPolicy|Are you sure you want to create merge request fot this policy?" +msgid "ScanExecutionPolicy|Are you sure you want to create merge request for this policy?" msgstr "" msgid "ScanExecutionPolicy|Back to edit policy" @@ -59074,8 +59080,10 @@ msgstr "" msgid "Unlimited" msgstr "" -msgid "UnlimitedMembersDuringTrialAlert|During your trial, invite as many members as you like to %{group_or_project} to collaborate with you." -msgstr "" +msgid "UnlimitedMembersDuringTrialAlert|During your trial, invite as many members as you like to %{name} to collaborate with you. When your trial ends, you'll have a maximum of %{limit} member on the Free tier, or you can get more by upgrading to a paid tier." +msgid_plural "UnlimitedMembersDuringTrialAlert|During your trial, invite as many members as you like to %{name} to collaborate with you. When your trial ends, you'll have a maximum of %{limit} members on the Free tier, or you can get more by upgrading to a paid tier." +msgstr[0] "" +msgstr[1] "" msgid "UnlimitedMembersDuringTrialAlert|Explore paid plans" msgstr "" @@ -62310,11 +62318,6 @@ msgstr "" msgid "When you transfer your project to a group, you can easily manage multiple projects, view usage quotas for storage, compute minutes, and users, and start a trial or upgrade to a paid tier." msgstr "" -msgid "When your trial ends, you'll have a maximum of %d member on the Free tier, or you can get more by upgrading to a paid tier." -msgid_plural "When your trial ends, you'll have a maximum of %d members on the Free tier, or you can get more by upgrading to a paid tier." -msgstr[0] "" -msgstr[1] "" - msgid "When your trial ends, you'll move to the Free tier, which has a limit of %{free_user_limit} seat. %{free_user_limit} seat will remain active, and members not occupying a seat will have the %{link_start}Over limit status%{link_end} and lose access to this group." msgid_plural "When your trial ends, you'll move to the Free tier, which has a limit of %{free_user_limit} seats. %{free_user_limit} seats will remain active, and members not occupying a seat will have the %{link_start}Over limit status%{link_end} and lose access to this group." msgstr[0] "" diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index 38dc83f303f..786eda444e3 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -412,6 +412,7 @@ RSpec.describe 'Database schema', # These pre-existing columns does not use a schema validation yet let(:ignored_jsonb_columns_map) do { + "Ai::Conversation::Message" => %w[extras error_details], "ApplicationSetting" => %w[repository_storages_weighted], "AlertManagement::Alert" => %w[payload], "Ci::BuildMetadata" => %w[config_options config_variables], diff --git a/spec/frontend/search/mock_data.js b/spec/frontend/search/mock_data.js index d355acaee11..c9ac817fea2 100644 --- a/spec/frontend/search/mock_data.js +++ b/spec/frontend/search/mock_data.js @@ -1739,15 +1739,60 @@ export const mockDataForBlobBody = { export const mockSourceBranches = [ { - text: 'master', - value: 'master', + text: 'Master Item', + value: 'master-item', }, { - text: 'feature', - value: 'feature', + text: 'Feature Item', + value: 'feature-item', }, { - text: 'develop', - value: 'develop', + text: 'Develop Item', + value: 'develop-item', + }, +]; + +export const mockAuthorsAxiosResponse = [ + { + id: 1, + username: 'root', + name: 'Administrator', + state: 'active', + locked: false, + avatar_url: + 'https://www.gravatar.com/avatar/8a2ba320206c6d79e89dd41a9081b7ae521d365f2054b3db1ac6462f692b176f?s=80&d=identicon', + web_url: 'http://127.0.0.1:3000/root', + status_tooltip_html: null, + show_status: false, + availability: null, + path: '/root', + }, + { + id: 65, + username: 'john', + name: 'John Doe', + state: 'active', + locked: false, + avatar_url: + 'https://www.gravatar.com/avatar/d9165b0da62fb9f9a57214a8fcc333101f2d10f494c662b53ffbeded3dcfa0dd?s=80&d=identicon', + web_url: 'http://127.0.0.1:3000/john', + status_tooltip_html: null, + show_status: false, + availability: null, + path: '/john', + }, + { + id: 50, + username: 'jane', + name: 'Jane Doe', + state: 'active', + locked: false, + avatar_url: + 'https://www.gravatar.com/avatar/224e81a612a566f3eb211d1d457b2335b662ad0dc7bb8d1b642056dd1b81755c?s=80&d=identicon', + web_url: 'http://127.0.0.1:3000/jane', + status_tooltip_html: null, + show_status: false, + availability: null, + path: '/jane', }, ]; diff --git a/spec/frontend/search/sidebar/components/author_filter_spec.js b/spec/frontend/search/sidebar/components/author_filter_spec.js new file mode 100644 index 00000000000..6f09f20a796 --- /dev/null +++ b/spec/frontend/search/sidebar/components/author_filter_spec.js @@ -0,0 +1,173 @@ +import { shallowMount } from '@vue/test-utils'; +import MockAdapter from 'axios-mock-adapter'; +import Vue, { nextTick } from 'vue'; +// eslint-disable-next-line no-restricted-imports +import Vuex from 'vuex'; +import { GlFormCheckbox } from '@gitlab/ui'; +import axios from '~/lib/utils/axios_utils'; +import AjaxCache from '~/lib/utils/ajax_cache'; +import AuthorFilter from '~/search/sidebar/components/author_filter/index.vue'; +import FilterDropdown from '~/search/sidebar/components/shared/filter_dropdown.vue'; +import { MOCK_QUERY, mockAuthorsAxiosResponse } from '../../mock_data'; + +Vue.use(Vuex); + +describe('Author filter', () => { + let wrapper; + const mock = new MockAdapter(axios); + + const actions = { + setQuery: jest.fn(), + applyQuery: jest.fn(), + }; + + const defaultState = { + query: { + scope: 'merge_requests', + group_id: 1, + search: '*', + }, + }; + + const createComponent = (state) => { + const store = new Vuex.Store({ + ...defaultState, + state, + actions, + }); + + wrapper = shallowMount(AuthorFilter, { + store, + }); + }; + + const findFilterDropdown = () => wrapper.findComponent(FilterDropdown); + const findGlFormCheckbox = () => wrapper.findComponent(GlFormCheckbox); + + beforeEach(() => { + createComponent(); + }); + + describe('when initial state', () => { + it('renders the component', () => { + expect(findFilterDropdown().exists()).toBe(true); + expect(findGlFormCheckbox().exists()).toBe(true); + }); + }); + + describe.each(['not[author_username]', 'author_username'])( + `when author is selected for %s author search`, + (authorParam) => { + beforeEach(async () => { + mock + .onGet('/-/autocomplete/users.json?current_user=true&active=true&search=') + .reply(200, mockAuthorsAxiosResponse); + createComponent({ + query: { + ...MOCK_QUERY, + [authorParam]: 'root', + }, + }); + + findFilterDropdown().vm.$emit('selected', 'root'); + await nextTick(); + }); + + it('renders the component with selected options', () => { + expect(findFilterDropdown().props('selectedItem')).toBe('root'); + expect(findGlFormCheckbox().attributes('checked')).toBe( + authorParam === 'not[author_username]' ? 'true' : undefined, + ); + }); + + it('displays the correct placeholder text and icon', () => { + expect(findFilterDropdown().props('searchText')).toBe('Administrator'); + expect(findFilterDropdown().props('icon')).toBe('user'); + }); + }, + ); + + describe('when opening dropdown', () => { + beforeEach(() => { + jest.spyOn(axios, 'get'); + jest.spyOn(AjaxCache, 'retrieve'); + + createComponent({ + groupInitialJson: { + id: 1, + full_name: 'gitlab-org/gitlab-test', + full_path: 'gitlab-org/gitlab-test', + }, + }); + }); + + afterEach(() => { + mock.restore(); + }); + + it('calls AjaxCache with correct params', () => { + findFilterDropdown().vm.$emit('shown'); + expect(AjaxCache.retrieve).toHaveBeenCalledWith( + '/-/autocomplete/users.json?current_user=true&active=true&group_id=1&search=', + ); + }); + }); + + describe.each([false, true])('when selecting an author with %s', (toggle) => { + beforeEach(() => { + createComponent({ + query: { + ...MOCK_QUERY, + }, + }); + }); + + it('calls setQuery with the correct params', () => { + const authorParam = 'author_username'; + const authorNotParam = 'not[author_username]'; + + wrapper.vm.toggleState = !toggle; + findFilterDropdown().vm.$emit('selected', 'root'); + + expect(actions.setQuery).toHaveBeenCalledTimes(2); + expect(actions.setQuery.mock.calls).toMatchObject([ + [ + expect.anything(), + { + key: toggle ? authorParam : authorNotParam, + value: 'root', + }, + ], + [ + expect.anything(), + { + key: toggle ? authorNotParam : authorParam, + value: '', + }, + ], + ]); + }); + }); + + describe('when resetting selected author', () => { + beforeEach(() => { + createComponent(); + }); + + it(`calls setQuery with correct param`, () => { + findFilterDropdown().vm.$emit('reset'); + + expect(actions.setQuery).toHaveBeenCalledWith(expect.anything(), { + key: 'author_username', + value: '', + }); + + expect(actions.setQuery).toHaveBeenCalledWith(expect.anything(), { + key: 'not[author_username]', + value: '', + }); + + expect(actions.applyQuery).toHaveBeenCalled(); + }); + }); +}); diff --git a/spec/frontend/search/sidebar/components/filter_dropdown_spec.js b/spec/frontend/search/sidebar/components/filter_dropdown_spec.js index 7582d67e8ef..c1bc724e45b 100644 --- a/spec/frontend/search/sidebar/components/filter_dropdown_spec.js +++ b/spec/frontend/search/sidebar/components/filter_dropdown_spec.js @@ -1,5 +1,5 @@ -import { shallowMount } from '@vue/test-utils'; import { GlCollapsibleListbox, GlListboxItem, GlIcon } from '@gitlab/ui'; +import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; import FilterDropdown from '~/search/sidebar/components/shared/filter_dropdown.vue'; import waitForPromises from 'helpers/wait_for_promises'; import { DEFAULT_DEBOUNCE_AND_THROTTLE_MS } from '~/lib/utils/constants'; @@ -11,7 +11,7 @@ describe('BranchDropdown', () => { const defaultProps = { listData: mockSourceBranches, - errors: [], + error: '', selectedItem: 'Master Item', headerText: 'Filter header', searchText: 'Search filter items', @@ -20,8 +20,8 @@ describe('BranchDropdown', () => { isLoading: false, }; - const createComponent = (props = {}, options = {}) => { - wrapper = shallowMount(FilterDropdown, { + const createComponent = (props = {}) => { + wrapper = shallowMountExtended(FilterDropdown, { propsData: { ...defaultProps, ...props, @@ -30,13 +30,12 @@ describe('BranchDropdown', () => { GlCollapsibleListbox, GlIcon, }, - ...options, }); }; const findGlCollapsibleListbox = () => wrapper.findComponent(GlCollapsibleListbox); const findGlListboxItems = () => wrapper.findAllComponents(GlListboxItem); - const findErrorMessages = () => wrapper.findAll('[data-testid="branch-dropdown-error-list"]'); + const findErrorMessage = () => wrapper.findByTestId('branch-dropdown-error'); describe('when nothing is selected', () => { beforeEach(() => { @@ -71,18 +70,24 @@ describe('BranchDropdown', () => { expect(props.resetButtonLabel).toBe('Reset'); }); - it('renders error messages when errors prop is passed', async () => { - const errors = ['Error 1', 'Error 2']; - createComponent({ errors }); + it('renders error message when error prop is passed', async () => { + createComponent({ error: 'Error 1' }); await waitForPromises(); + expect(findErrorMessage().exists()).toBe(true); + expect(findErrorMessage().text()).toBe('Error 1'); + }); - const errorMessages = findErrorMessages(); + it('renders error message reactivly', async () => { + createComponent(); - expect(errorMessages.length).toBe(errors.length); - errorMessages.wrappers.forEach((errorWrapper, index) => { - expect(errorWrapper.text()).toContain(errors[index]); - }); + await waitForPromises(); + expect(findErrorMessage().exists()).toBe(false); + + wrapper.setProps({ error: 'Error 1' }); + await waitForPromises(); + expect(findErrorMessage().exists()).toBe(true); + expect(findErrorMessage().text()).toBe('Error 1'); }); it('search filters items', async () => { diff --git a/spec/frontend/search/sidebar/components/merge_requests_filters_spec.js b/spec/frontend/search/sidebar/components/merge_requests_filters_spec.js index 5c14fc13170..4721fba09d3 100644 --- a/spec/frontend/search/sidebar/components/merge_requests_filters_spec.js +++ b/spec/frontend/search/sidebar/components/merge_requests_filters_spec.js @@ -8,6 +8,7 @@ import StatusFilter from '~/search/sidebar/components/status_filter/index.vue'; import ArchivedFilter from '~/search/sidebar/components/archived_filter/index.vue'; import SourceBranchFilter from '~/search/sidebar/components/source_branch_filter/index.vue'; import LabelFilter from '~/search/sidebar/components/label_filter/index.vue'; +import AuthorFilter from '~/search/sidebar/components/author_filter/index.vue'; Vue.use(Vuex); @@ -21,7 +22,12 @@ describe('GlobalSearch MergeRequestsFilters', () => { const createComponent = ( initialState = {}, - provide = { glFeatures: { searchMrFilterSourceBranch: true } }, + provide = { + glFeatures: { + searchMrFilterSourceBranch: true, + searchMrFilterAuthor: true, + }, + }, ) => { const store = new Vuex.Store({ state: { @@ -45,8 +51,9 @@ describe('GlobalSearch MergeRequestsFilters', () => { const findArchivedFilter = () => wrapper.findComponent(ArchivedFilter); const findSourceBranchFilter = () => wrapper.findComponent(SourceBranchFilter); const findLabelFilter = () => wrapper.findComponent(LabelFilter); + const findAuthorFilter = () => wrapper.findComponent(AuthorFilter); - describe('Renders correctly with Archived Filter', () => { + describe('When renders correctly with Archived Filter', () => { beforeEach(() => { createComponent(); }); @@ -59,16 +66,20 @@ describe('GlobalSearch MergeRequestsFilters', () => { expect(findArchivedFilter().exists()).toBe(true); }); - it('renders sourceBranchFilter', () => { + it('renders SourceBranchFilter', () => { expect(findSourceBranchFilter().exists()).toBe(true); }); - it('renders label filter', () => { + it('renders LabelFilter', () => { expect(findLabelFilter().exists()).toBe(true); }); + + it('renders AuthorFilter', () => { + expect(findAuthorFilter().exists()).toBe(true); + }); }); - describe('Renders correctly with basic search', () => { + describe('When renders correctly with basic search', () => { beforeEach(() => { createComponent({ searchType: 'basic' }); }); @@ -81,35 +92,46 @@ describe('GlobalSearch MergeRequestsFilters', () => { expect(findArchivedFilter().exists()).toBe(true); }); - it('renders sourceBranchFilter', () => { + it('renders SourceBranchFilter', () => { expect(findSourceBranchFilter().exists()).toBe(true); }); - it('will not render label filter', () => { + it('will not render LabelFilter', () => { expect(findLabelFilter().exists()).toBe(false); }); + + it('will not render AuthorFilter', () => { + expect(findAuthorFilter().exists()).toBe(false); + }); }); - describe.each([true, false])( - 'When feature flag search_mr_filter_source_branch is', - (searchMrFilterSourceBranch) => { - beforeEach(() => { - createComponent(null, { glFeatures: { searchMrFilterSourceBranch } }); - }); + describe('When feature flag search_mr_filter_source_branch is disabled', () => { + beforeEach(() => { + createComponent(null, { glFeatures: { searchMrFilterSourceBranch: false } }); + }); - it(`${searchMrFilterSourceBranch ? 'will' : 'will not'} render sourceBranchFilter`, () => { - expect(findSourceBranchFilter().exists()).toBe(searchMrFilterSourceBranch); - }); - }, - ); + it(`will not render SourceBranchFilter`, () => { + expect(findSourceBranchFilter().exists()).toBe(false); + }); + }); - describe('hasMissingProjectContext getter', () => { + describe('When feature flag search_mr_filter_author is disabled', () => { + beforeEach(() => { + createComponent(null, { glFeatures: { searchMrFilterAuthor: false } }); + }); + + it(`will not render AuthorFilter`, () => { + expect(findAuthorFilter().exists()).toBe(false); + }); + }); + + describe('#hasMissingProjectContext getter', () => { beforeEach(() => { defaultGetters.hasMissingProjectContext = () => false; createComponent(); }); - it('hides archived filter', () => { + it('hides ArchivedFilter', () => { expect(findArchivedFilter().exists()).toBe(false); }); }); diff --git a/spec/graphql/types/user_type_spec.rb b/spec/graphql/types/user_type_spec.rb index 3fa592f9160..e55da9aaf09 100644 --- a/spec/graphql/types/user_type_spec.rb +++ b/spec/graphql/types/user_type_spec.rb @@ -62,6 +62,7 @@ RSpec.describe GitlabSchema.types['User'], feature_category: :user_profile do pronouns ide userPreferences + type ] expect(described_class).to include_graphql_fields(*expected_fields) @@ -397,4 +398,56 @@ RSpec.describe GitlabSchema.types['User'], feature_category: :user_profile do is_expected.to have_graphql_type(Types::UserPreferencesType) end end + + describe 'type field' do + subject { described_class.fields['type'] } + + let_it_be(:admin) { create(:user, :admin) } + let_it_be(:regular_user) { create(:user) } + let_it_be(:placeholder_user) { create(:user, :placeholder) } + let_it_be(:import_user) { create(:user, :import_user) } + let_it_be(:ghost_user) { create(:user, :ghost) } + + let(:query) do + <<~GQL + query($id: UserID!) { + user(id: $id) { + type + } + } + GQL + end + + it 'returns type field' do + is_expected.to have_graphql_type(Types::Users::TypeEnum.to_non_null_type) + end + + it 'returns HUMAN for regular users' do + result = GitlabSchema.execute(query, variables: { id: regular_user.to_global_id.to_s }, + context: { current_user: admin }).as_json + + expect(result.dig('data', 'user', 'type')).to eq('HUMAN') + end + + it 'returns PLACEHOLDER for placeholder users' do + result = GitlabSchema.execute(query, variables: { id: placeholder_user.to_global_id.to_s }, + context: { current_user: admin }).as_json + + expect(result.dig('data', 'user', 'type')).to eq('PLACEHOLDER') + end + + it 'returns IMPORT_USER for import users' do + result = GitlabSchema.execute(query, variables: { id: import_user.to_global_id.to_s }, + context: { current_user: admin }).as_json + + expect(result.dig('data', 'user', 'type')).to eq('IMPORT_USER') + end + + it 'returns GHOST for ghost users' do + result = GitlabSchema.execute(query, variables: { id: ghost_user.to_global_id.to_s }, + context: { current_user: admin }).as_json + + expect(result.dig('data', 'user', 'type')).to eq('GHOST') + end + end end diff --git a/spec/lib/gitlab/background_migration/backfill_protected_branch_merge_access_levels_protected_branch_namespace_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_protected_branch_merge_access_levels_protected_branch_namespace_id_spec.rb new file mode 100644 index 00000000000..34f5f5d8401 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_protected_branch_merge_access_levels_protected_branch_namespace_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillProtectedBranchMergeAccessLevelsProtectedBranchNamespaceId, + feature_category: :source_code_management, + schema: 20241204130226 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :protected_branch_merge_access_levels } + let(:backfill_column) { :protected_branch_namespace_id } + let(:backfill_via_table) { :protected_branches } + let(:backfill_via_column) { :namespace_id } + let(:backfill_via_foreign_key) { :protected_branch_id } + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_protected_branch_merge_access_levels_protected_branch_project_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_protected_branch_merge_access_levels_protected_branch_project_id_spec.rb new file mode 100644 index 00000000000..321b4af4783 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_protected_branch_merge_access_levels_protected_branch_project_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillProtectedBranchMergeAccessLevelsProtectedBranchProjectId, + feature_category: :source_code_management, + schema: 20241204130221 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :protected_branch_merge_access_levels } + let(:backfill_column) { :protected_branch_project_id } + let(:backfill_via_table) { :protected_branches } + let(:backfill_via_column) { :project_id } + let(:backfill_via_foreign_key) { :protected_branch_id } + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_status_page_published_incidents_namespace_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_status_page_published_incidents_namespace_id_spec.rb new file mode 100644 index 00000000000..7d26491ae32 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_status_page_published_incidents_namespace_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillStatusPagePublishedIncidentsNamespaceId, + feature_category: :incident_management, + schema: 20241205143056 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :status_page_published_incidents } + let(:backfill_column) { :namespace_id } + let(:backfill_via_table) { :issues } + let(:backfill_via_column) { :namespace_id } + let(:backfill_via_foreign_key) { :issue_id } + end +end diff --git a/spec/lib/gitlab/ci/tags/bulk_insert_spec.rb b/spec/lib/gitlab/ci/tags/bulk_insert_spec.rb index 32b4235f2c4..93ed0912731 100644 --- a/spec/lib/gitlab/ci/tags/bulk_insert_spec.rb +++ b/spec/lib/gitlab/ci/tags/bulk_insert_spec.rb @@ -2,208 +2,223 @@ require 'spec_helper' -RSpec.describe Gitlab::Ci::Tags::BulkInsert do +RSpec.describe Gitlab::Ci::Tags::BulkInsert, feature_category: :continuous_integration do + using RSpec::Parameterized::TableSyntax + let_it_be(:project) { create(:project, :repository) } let_it_be(:pipeline) { create(:ci_pipeline, project: project) } let_it_be_with_refind(:job) { create(:ci_build, :unique_name, pipeline: pipeline) } let_it_be_with_refind(:other_job) { create(:ci_build, :unique_name, pipeline: pipeline) } - let(:statuses) { [job, other_job] } + let_it_be_with_refind(:runner) { create(:ci_runner) } + let_it_be_with_refind(:other_runner) { create(:ci_runner, :project_type, projects: [project]) } + + let(:statuses) { [taggable, other_taggable] } let(:config) { described_class::NoConfig.new } subject(:service) { described_class.new(statuses, config: config) } - describe '.bulk_insert_tags!' do - let(:inserter) { instance_double(described_class) } - - it 'delegates to bulk insert class' do - expect(described_class) - .to receive(:new) - .with(statuses, config: nil) - .and_return(inserter) - - expect(inserter).to receive(:insert!) - - described_class.bulk_insert_tags!(statuses) - end + where(:taggable_class, :taggable, :other_taggable, :tagging_class, :taggable_id_column, :partition_column, + :expected_configuration) do + Ci::Build | ref(:job) | ref(:other_job) | Ci::BuildTag | :build_id | :partition_id | + described_class::BuildsTagsConfiguration + Ci::Runner | ref(:runner) | ref(:other_runner) | Ci::RunnerTagging | :runner_id | :runner_type | + described_class::RunnerTaggingsConfiguration end - describe '#insert!' do - context 'without tags' do - it { expect(service.insert!).to be_truthy } + with_them do + describe '.bulk_insert_tags!' do + let(:inserter) { instance_double(described_class) } + + it 'delegates to bulk insert class' do + expect(described_class) + .to receive(:new) + .with(statuses, config: nil) + .and_return(inserter) + + expect(inserter).to receive(:insert!) + + described_class.bulk_insert_tags!(statuses) + end end - context 'with tags' do - before do - job.tag_list = %w[tag1 tag2] - other_job.tag_list = %w[tag2 tag3 tag4] + describe '#insert!' do + context 'without tags' do + it { expect(service.insert!).to be_truthy } end - it 'persists tags' do - expect(service.insert!).to be_truthy - - expect(job.reload.tag_list).to match_array(%w[tag1 tag2]) - expect(other_job.reload.tag_list).to match_array(%w[tag2 tag3 tag4]) - end - - it 'persists taggings' do - service.insert! - - expect(job.taggings.size).to eq(2) - expect(other_job.taggings.size).to eq(3) - - expect(Ci::Build.tagged_with('tag1')).to include(job) - expect(Ci::Build.tagged_with('tag2')).to include(job, other_job) - expect(Ci::Build.tagged_with('tag3')).to include(other_job) - end - - it 'strips tags' do - job.tag_list = [' taga', 'tagb ', ' tagc '] - - service.insert! - expect(job.tags.map(&:name)).to match_array(%w[taga tagb tagc]) - end - - context 'when batching inserts for tags' do + context 'with tags' do before do - stub_const("#{described_class}::TAGS_BATCH_SIZE", 2) + taggable.tag_list = %w[tag1 tag2] + other_taggable.tag_list = %w[tag2 tag3 tag4] end - it 'inserts tags in batches' do - recorder = ActiveRecord::QueryRecorder.new { service.insert! } - count = recorder.log.count { |query| query.include?('INSERT INTO "tags"') } + it 'persists tags' do + expect(service.insert!).to be_truthy - expect(count).to eq(2) - end - end - - context 'when batching inserts for taggings' do - before do - stub_const("#{described_class}::TAGGINGS_BATCH_SIZE", 2) + expect(taggable.reload.tag_list).to match_array(%w[tag1 tag2]) + expect(other_taggable.reload.tag_list).to match_array(%w[tag2 tag3 tag4]) end - it 'inserts taggings in batches' do - recorder = ActiveRecord::QueryRecorder.new { service.insert! } - count = recorder.log.count { |query| query.include?('INSERT INTO "taggings"') } - - expect(count).to eq(3) - end - end - - context 'with no config provided' do - it 'does not persist tag links' do + it 'persists taggings' do service.insert! - expect(job.tag_links).to be_empty - expect(other_job.tag_links).to be_empty - end - end + expect(taggable.taggings.size).to eq(2) + expect(other_taggable.taggings.size).to eq(3) - context 'with config provided by the factory' do - let(:config) { nil } - - it 'generates a valid config' do - expect(service.config).to be_a(described_class::BuildsTagsConfiguration) + expect(taggable_class.tagged_with('tag1')).to include(taggable) + expect(taggable_class.tagged_with('tag2')).to include(taggable, other_taggable) + expect(taggable_class.tagged_with('tag3')).to include(other_taggable) end - context 'with flags' do + it 'strips tags' do + taggable.tag_list = [' taga', 'tagb ', ' tagc '] + + service.insert! + expect(taggable.tags.map(&:name)).to match_array(%w[taga tagb tagc]) + end + + context 'when batching inserts for tags' do before do - allow(service.config).to receive(:monomorphic_taggings?) { monomorphic_taggings } - allow(service.config).to receive(:polymorphic_taggings?) { polymorphic_taggings } + stub_const("#{described_class}::TAGS_BATCH_SIZE", 2) end - context 'when writing to both tables' do - let(:monomorphic_taggings) { true } - let(:polymorphic_taggings) { true } + it 'inserts tags in batches' do + recorder = ActiveRecord::QueryRecorder.new { service.insert! } + count = recorder.log.count { |query| query.include?('INSERT INTO "tags"') } - it 'persists tag links and taggings' do - service.insert! + expect(count).to eq(2) + end + end - expect(job.tag_links).not_to be_empty - expect(other_job.tag_links).not_to be_empty + context 'when batching inserts for taggings' do + before do + stub_const("#{described_class}::TAGGINGS_BATCH_SIZE", 2) + end - expect(jobs_tagged_with('tag1')).to contain_exactly(job) - expect(jobs_tagged_with('tag2')).to contain_exactly(job, other_job) - expect(jobs_tagged_with('tag3')).to contain_exactly(other_job) + it 'inserts taggings in batches' do + recorder = ActiveRecord::QueryRecorder.new { service.insert! } + count = recorder.log.count { |query| query.include?('INSERT INTO "taggings"') } - expect(job.taggings).not_to be_empty - expect(other_job.taggings).not_to be_empty + expect(count).to eq(3) + end + end - expect(Ci::Build.tagged_with('tag1')).to contain_exactly(job) - expect(Ci::Build.tagged_with('tag2')).to contain_exactly(job, other_job) - expect(Ci::Build.tagged_with('tag3')).to contain_exactly(other_job) + context 'with no config provided' do + it 'does not persist tag links' do + service.insert! + + expect(taggable.tag_links).to be_empty + expect(other_taggable.tag_links).to be_empty + end + end + + context 'with config provided by the factory' do + let(:config) { nil } + + it 'generates a valid config' do + expect(service.config).to be_a(expected_configuration) + end + + context 'with flags' do + before do + allow(service.config).to receive(:monomorphic_taggings?) { monomorphic_taggings } + allow(service.config).to receive(:polymorphic_taggings?) { polymorphic_taggings } end - end - context 'when writing only to taggings' do - let(:monomorphic_taggings) { false } - let(:polymorphic_taggings) { true } + context 'when writing to both tables' do + let(:monomorphic_taggings) { true } + let(:polymorphic_taggings) { true } - it 'persists taggings' do - service.insert! + it 'persists tag links and taggings' do + service.insert! - expect(job.tag_links).to be_empty - expect(other_job.tag_links).to be_empty + expect(taggable.tag_links).not_to be_empty + expect(other_taggable.tag_links).not_to be_empty - expect(job.taggings).not_to be_empty - expect(other_job.taggings).not_to be_empty + expect(tagged_with('tag1')).to contain_exactly(taggable) + expect(tagged_with('tag2')).to contain_exactly(taggable, other_taggable) + expect(tagged_with('tag3')).to contain_exactly(other_taggable) - expect(Ci::Build.tagged_with('tag1')).to contain_exactly(job) - expect(Ci::Build.tagged_with('tag2')).to contain_exactly(job, other_job) - expect(Ci::Build.tagged_with('tag3')).to contain_exactly(other_job) + expect(taggable.taggings).not_to be_empty + expect(other_taggable.taggings).not_to be_empty + + expect(taggable_class.tagged_with('tag1')).to contain_exactly(taggable) + expect(taggable_class.tagged_with('tag2')).to contain_exactly(taggable, other_taggable) + expect(taggable_class.tagged_with('tag3')).to contain_exactly(other_taggable) + end end - end - context 'when writing only to link table' do - let(:monomorphic_taggings) { true } - let(:polymorphic_taggings) { false } + context 'when writing only to taggings' do + let(:monomorphic_taggings) { false } + let(:polymorphic_taggings) { true } - it 'persists tag links' do - service.insert! + it 'persists taggings' do + service.insert! - expect(job.tag_links).not_to be_empty - expect(other_job.tag_links).not_to be_empty + expect(taggable.tag_links).to be_empty + expect(other_taggable.tag_links).to be_empty - expect(jobs_tagged_with('tag1')).to contain_exactly(job) - expect(jobs_tagged_with('tag2')).to contain_exactly(job, other_job) - expect(jobs_tagged_with('tag3')).to contain_exactly(other_job) + expect(taggable.taggings).not_to be_empty + expect(other_taggable.taggings).not_to be_empty - expect(job.taggings).to be_empty - expect(other_job.taggings).to be_empty + expect(taggable_class.tagged_with('tag1')).to contain_exactly(taggable) + expect(taggable_class.tagged_with('tag2')).to contain_exactly(taggable, other_taggable) + expect(taggable_class.tagged_with('tag3')).to contain_exactly(other_taggable) + end end - end - def jobs_tagged_with(tag) - scope = Ci::BuildTag - .where(tag_id: Ci::Tag.where(name: tag)) - .where(Ci::BuildTag.arel_table[:build_id].eq(Ci::Build.arel_table[:id])) - .where(Ci::BuildTag.arel_table[:partition_id].eq(Ci::Build.arel_table[:partition_id])) + context 'when writing only to link table' do + let(:monomorphic_taggings) { true } + let(:polymorphic_taggings) { false } - Ci::Build.where_exists(scope) + it 'persists tag links' do + service.insert! + + expect(taggable.tag_links).not_to be_empty + expect(other_taggable.tag_links).not_to be_empty + + expect(tagged_with('tag1')).to contain_exactly(taggable) + expect(tagged_with('tag2')).to contain_exactly(taggable, other_taggable) + expect(tagged_with('tag3')).to contain_exactly(other_taggable) + + expect(taggable.taggings).to be_empty + expect(other_taggable.taggings).to be_empty + end + end + + def tagged_with(tag) + scope = tagging_class + .where(tag_id: Ci::Tag.where(name: tag)) + .where(tagging_class.arel_table[taggable_id_column].eq(taggable_class.arel_table[:id])) + .where(tagging_class.arel_table[partition_column].eq(taggable_class.arel_table[partition_column])) + + taggable_class.where_exists(scope) + end end end end - end - context 'with tags for only one job' do - before do - job.tag_list = %w[tag1 tag2] - end + context 'with tags for only one taggable' do + before do + taggable.tag_list = %w[tag1 tag2] + end - it 'persists tags' do - expect(service.insert!).to be_truthy + it 'persists tags' do + expect(service.insert!).to be_truthy - expect(job.reload.tag_list).to match_array(%w[tag1 tag2]) - expect(other_job.reload.tag_list).to be_empty - end + expect(taggable.reload.tag_list).to match_array(%w[tag1 tag2]) + expect(other_taggable.reload.tag_list).to be_empty + end - it 'persists taggings' do - service.insert! + it 'persists taggings' do + service.insert! - expect(job.taggings.size).to eq(2) + expect(taggable.taggings.size).to eq(2) - expect(Ci::Build.tagged_with('tag1')).to include(job) - expect(Ci::Build.tagged_with('tag2')).to include(job) + expect(taggable_class.tagged_with('tag1')).to include(taggable) + expect(taggable_class.tagged_with('tag2')).to include(taggable) + end end end end diff --git a/spec/lib/gitlab/database/sharding_key_spec.rb b/spec/lib/gitlab/database/sharding_key_spec.rb index 70551fd8f72..483d4c4ce7e 100644 --- a/spec/lib/gitlab/database/sharding_key_spec.rb +++ b/spec/lib/gitlab/database/sharding_key_spec.rb @@ -192,8 +192,6 @@ RSpec.describe 'new tables missing sharding_key', feature_category: :cell do "projects" => 'https://gitlab.com/gitlab-org/gitlab/-/issues/476211', "push_rules" => 'https://gitlab.com/gitlab-org/gitlab/-/issues/476212', "snippets" => 'https://gitlab.com/gitlab-org/gitlab/-/issues/476216', - "upcoming_reconciliations" => 'https://gitlab.com/gitlab-org/gitlab/-/issues/476217', - "vulnerability_exports" => 'https://gitlab.com/gitlab-org/gitlab/-/issues/476219', "topics" => 'https://gitlab.com/gitlab-org/gitlab/-/issues/463254', "oauth_access_tokens" => "https://gitlab.com/gitlab-org/gitlab/-/issues/496717", "oauth_access_grants" => "https://gitlab.com/gitlab-org/gitlab/-/issues/496717", diff --git a/spec/migrations/20241204130225_queue_backfill_protected_branch_merge_access_levels_project_id_spec.rb b/spec/migrations/20241204130225_queue_backfill_protected_branch_merge_access_levels_project_id_spec.rb new file mode 100644 index 00000000000..62750afc635 --- /dev/null +++ b/spec/migrations/20241204130225_queue_backfill_protected_branch_merge_access_levels_project_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillProtectedBranchMergeAccessLevelsProjectId, feature_category: :source_code_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :protected_branch_merge_access_levels, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :protected_branch_project_id, + :protected_branches, + :project_id, + :protected_branch_id + ] + ) + } + end + end +end diff --git a/spec/migrations/20241204130230_queue_backfill_protected_branch_merge_access_levels_namespace_id_spec.rb b/spec/migrations/20241204130230_queue_backfill_protected_branch_merge_access_levels_namespace_id_spec.rb new file mode 100644 index 00000000000..f92e214bd33 --- /dev/null +++ b/spec/migrations/20241204130230_queue_backfill_protected_branch_merge_access_levels_namespace_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillProtectedBranchMergeAccessLevelsNamespaceId, feature_category: :source_code_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :protected_branch_merge_access_levels, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :protected_branch_namespace_id, + :protected_branches, + :namespace_id, + :protected_branch_id + ] + ) + } + end + end +end diff --git a/spec/migrations/20241205143060_queue_backfill_status_page_published_incidents_namespace_id_spec.rb b/spec/migrations/20241205143060_queue_backfill_status_page_published_incidents_namespace_id_spec.rb new file mode 100644 index 00000000000..07809ba2f66 --- /dev/null +++ b/spec/migrations/20241205143060_queue_backfill_status_page_published_incidents_namespace_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillStatusPagePublishedIncidentsNamespaceId, feature_category: :incident_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :status_page_published_incidents, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :namespace_id, + :issues, + :namespace_id, + :issue_id + ] + ) + } + end + end +end diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index 0093f7469e5..64b02c3aca1 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -484,25 +484,32 @@ RSpec.describe Ci::Pipeline, :mailer, factory_default: :keep, feature_category: end end - describe '.created_after' do + context 'with created filters' do let_it_be(:old_pipeline) { create(:ci_pipeline, created_at: 1.week.ago) } - let_it_be(:pipeline) { create(:ci_pipeline) } - - subject { described_class.created_after(1.day.ago) } - - it 'returns the pipeline' do - is_expected.to contain_exactly(pipeline) - end - end - - describe '.created_before_id' do - let_it_be(:pipeline) { create(:ci_pipeline) } let_it_be(:new_pipeline) { create(:ci_pipeline) } - subject { described_class.created_before_id(new_pipeline.id) } + describe '.created_after' do + subject { described_class.created_after(1.day.ago) } - it 'returns the pipeline' do - is_expected.to contain_exactly(pipeline) + it 'returns the newer pipeline' do + is_expected.to contain_exactly(new_pipeline) + end + end + + describe '.created_before' do + subject { described_class.created_before(1.day.ago) } + + it 'returns the older pipeline' do + is_expected.to contain_exactly(old_pipeline) + end + end + + describe '.created_before_id' do + subject { described_class.created_before_id(new_pipeline.id) } + + it 'returns the pipeline' do + is_expected.to contain_exactly(old_pipeline) + end end end diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb index 19a9e70df36..62d1023c797 100644 --- a/spec/models/ci/runner_spec.rb +++ b/spec/models/ci/runner_spec.rb @@ -10,6 +10,24 @@ RSpec.describe Ci::Runner, type: :model, factory_default: :keep, feature_categor let_it_be(:project) { create(:project, group: group) } let_it_be(:other_project) { create(:project) } + describe 'associations' do + it { is_expected.to belong_to(:creator).class_name('User').optional } + + it { is_expected.to have_many(:runner_managers).inverse_of(:runner) } + it { is_expected.to have_many(:builds) } + it { is_expected.to have_one(:last_build).class_name('Ci::Build') } + it { is_expected.to have_many(:running_builds).inverse_of(:runner) } + + it { is_expected.to have_many(:runner_projects).inverse_of(:runner) } + it { is_expected.to have_many(:projects).through(:runner_projects) } + + it { is_expected.to have_many(:runner_namespaces).inverse_of(:runner) } + it { is_expected.to have_many(:groups).through(:runner_namespaces) } + it { is_expected.to have_one(:owner_runner_namespace).class_name('Ci::RunnerNamespace') } + + it { is_expected.to have_many(:tag_links).class_name('Ci::RunnerTagging').inverse_of(:runner) } + end + it_behaves_like 'having unique enum values' it_behaves_like 'it has loose foreign keys' do @@ -61,7 +79,7 @@ RSpec.describe Ci::Runner, type: :model, factory_default: :keep, feature_categor let(:tag_name) { 'tag123' } context 'on save' do - let_it_be_with_reload(:runner) { create(:ci_runner) } + let(:runner) { create(:ci_runner, :group, groups: [group]) } before do runner.tag_list = [tag_name] @@ -96,6 +114,41 @@ RSpec.describe Ci::Runner, type: :model, factory_default: :keep, feature_categor expect(described_class.tagged_with(tag_name)).to include(runner) end end + + context 'when runner is not yet synced to partitioned table' do + let(:connection) { Ci::ApplicationRecord.connection } + + before do + # Simulate legacy runners not present in sharded table (created when FK was not present) + runner + + connection.execute(<<~SQL) + DELETE FROM ci_runners_e59bb2812d; + SQL + end + + context 'tag does not exist' do + before do + runner.tag_list = [tag_name] + end + + it 'creates a tag and syncs runner to partitioned table' do + expect { runner.save! } + .to change(Ci::Tag, :count).by(1) + .and change { partitioned_runner_exists?(runner) }.from(false).to(true) + end + end + + private + + def partitioned_runner_exists?(runner) + result = connection.execute(<<~SQL) + SELECT COUNT(*) FROM ci_runners_e59bb2812d WHERE id = #{runner.id}; + SQL + + result.first['count'].positive? + end + end end end @@ -1229,6 +1282,7 @@ RSpec.describe Ci::Runner, type: :model, factory_default: :keep, feature_categor expect(runner.tags.count).to eq(1) expect(runner.tags.first.name).to eq('tag') + expect(runner.tag_links.count).to eq(1) end it 'strips tags' do @@ -1260,6 +1314,20 @@ RSpec.describe Ci::Runner, type: :model, factory_default: :keep, feature_categor end end end + + context 'with write_to_ci_runner_taggings disabled' do + before do + stub_feature_flags(write_to_ci_runner_taggings: false) + end + + it 'does not save tag_links' do + runner.save! + + expect(runner.tags.count).to eq(1) + expect(runner.tags.first.name).to eq('tag') + expect(runner.tag_links).to be_empty + end + end end describe '#has_tags?' do diff --git a/spec/models/project_ci_cd_setting_spec.rb b/spec/models/project_ci_cd_setting_spec.rb index 85e43df5a3f..73b740bc3cd 100644 --- a/spec/models/project_ci_cd_setting_spec.rb +++ b/spec/models/project_ci_cd_setting_spec.rb @@ -43,15 +43,6 @@ RSpec.describe ProjectCiCdSetting, feature_category: :continuous_integration do end end - describe '.configured_to_delete_old_pipelines' do - let_it_be(:project) { create(:project, ci_delete_pipelines_in_seconds: 2.weeks.to_i) } - let_it_be(:other_project) { create(:project, group_runners_enabled: true) } - - it 'includes settings with values present' do - expect(described_class.configured_to_delete_old_pipelines).to contain_exactly(project.ci_cd_settings) - end - end - describe '#pipeline_variables_minimum_override_role' do it 'is maintainer by default' do expect(described_class.new.pipeline_variables_minimum_override_role).to eq('maintainer') @@ -138,4 +129,13 @@ RSpec.describe ProjectCiCdSetting, feature_category: :continuous_integration do end end end + + describe '.configured_to_delete_old_pipelines' do + let_it_be(:project) { create(:project, ci_delete_pipelines_in_seconds: 2.weeks.to_i) } + let_it_be(:other_project) { create(:project, group_runners_enabled: true) } + + it 'includes settings with values present' do + expect(described_class.configured_to_delete_old_pipelines).to contain_exactly(project.ci_cd_settings) + end + end end diff --git a/spec/models/users/credit_card_validation_spec.rb b/spec/models/users/credit_card_validation_spec.rb index 0bf2fd6f48d..ab3b7f008af 100644 --- a/spec/models/users/credit_card_validation_spec.rb +++ b/spec/models/users/credit_card_validation_spec.rb @@ -335,14 +335,6 @@ RSpec.describe Users::CreditCardValidation, feature_category: :user_profile do end it { is_expected.to eq(true) } - - context 'when the feature flag is disabled' do - before do - stub_feature_flags(credit_card_validation_daily_limit: false) - end - - it { is_expected.to eq(false) } - end end context 'when the limit is exceeded but records have credit_card_validated_at > 24 hours' do diff --git a/spec/support/shared_examples/graphql/types/merge_request_interactions_type_shared_examples.rb b/spec/support/shared_examples/graphql/types/merge_request_interactions_type_shared_examples.rb index a516e22015d..14edf9cc438 100644 --- a/spec/support/shared_examples/graphql/types/merge_request_interactions_type_shared_examples.rb +++ b/spec/support/shared_examples/graphql/types/merge_request_interactions_type_shared_examples.rb @@ -58,6 +58,7 @@ RSpec.shared_examples "a user type with merge request interaction type" do pronouns ide userPreferences + type ] # TODO: 'workspaces' needs to be included, but only when this spec is run in EE context, to account for the diff --git a/spec/workers/ci/destroy_old_pipelines_worker_spec.rb b/spec/workers/ci/destroy_old_pipelines_worker_spec.rb new file mode 100644 index 00000000000..84dff9de7e5 --- /dev/null +++ b/spec/workers/ci/destroy_old_pipelines_worker_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::DestroyOldPipelinesWorker, :clean_gitlab_redis_shared_state, feature_category: :continuous_integration do + let_it_be(:project) { create(:project, ci_delete_pipelines_in_seconds: 2.weeks.to_i) } + let_it_be(:ancient_pipeline) { create(:ci_pipeline, project: project, created_at: 1.year.ago) } + let_it_be(:old_pipeline) { create(:ci_pipeline, project: project, created_at: 1.month.ago) } + let_it_be(:new_pipeline) { create(:ci_pipeline, project: project, created_at: 1.week.ago) } + + describe '#perform' do + before do + Gitlab::Redis::SharedState.with do |redis| + redis.rpush(Ci::ScheduleOldPipelinesRemovalCronWorker::QUEUE_KEY, [project.id]) + end + end + + subject(:perform) { described_class.new.perform_work } + + it 'destroys the configured amount of pipelines' do + stub_const("#{described_class.name}::LIMIT", 1) + + expect { perform }.to change { project.all_pipelines.count }.by(-1) + expect(new_pipeline.reload).to be_present + end + + it 'loops thought the available pipelines' do + stub_const("#{described_class.name}::LIMIT", 3) + + expect { perform }.to change { project.all_pipelines.count }.by(-2) + expect(new_pipeline.reload).to be_present + end + + it_behaves_like 'an idempotent worker' do + let(:job_args) { [project.id] } + + it 'executes the service' do + expect { perform }.not_to raise_error + end + end + end +end diff --git a/spec/workers/ci/schedule_old_pipelines_removal_cron_worker_spec.rb b/spec/workers/ci/schedule_old_pipelines_removal_cron_worker_spec.rb new file mode 100644 index 00000000000..d3cb05e68dd --- /dev/null +++ b/spec/workers/ci/schedule_old_pipelines_removal_cron_worker_spec.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::ScheduleOldPipelinesRemovalCronWorker, + :clean_gitlab_redis_shared_state, feature_category: :continuous_integration do + let(:worker) { described_class.new } + + let_it_be(:project) { create(:project, ci_delete_pipelines_in_seconds: 2.weeks.to_i) } + + it { is_expected.to include_module(CronjobQueue) } + it { expect(described_class.idempotent?).to be_truthy } + + describe '#perform' do + it 'enqueues DestroyOldPipelinesWorker jobs' do + expect(Ci::DestroyOldPipelinesWorker).to receive(:perform_with_capacity) + + worker.perform + end + + it 'enqueues projects to be processed' do + worker.perform + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.lpop(described_class::QUEUE_KEY).to_i).to eq(project.id) + end + end + + context 'when the worker reaches the maximum number of records per execution' do + before do + stub_const("#{described_class}::PROJECTS_LIMIT", 1) + end + + it 'sets the last processed record id in Redis cache' do + worker.perform + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.get(described_class::LAST_PROCESSED_REDIS_KEY).to_i).to eq(project.id) + end + end + end + + context 'when the worker continues processing from previous execution' do + let_it_be(:other_project) { create(:project, ci_delete_pipelines_in_seconds: 2.weeks.to_i) } + + before do + Gitlab::Redis::SharedState.with do |redis| + redis.set(described_class::LAST_PROCESSED_REDIS_KEY, other_project.id) + end + end + + it 'enqueues projects to be processed' do + worker.perform + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.lpop(described_class::QUEUE_KEY).to_i).to eq(other_project.id) + end + end + + it 'enqueues DestroyOldPipelinesWorker jobs' do + expect(Ci::DestroyOldPipelinesWorker).to receive(:perform_with_capacity) + + worker.perform + end + end + + context 'when the worker finishes processing before running out of batches' do + before do + stub_const("#{described_class}::PROJECTS_LIMIT", 2) + + Gitlab::Redis::SharedState.with do |redis| + redis.set(described_class::LAST_PROCESSED_REDIS_KEY, 0) + end + end + + it 'clears the last processed record id in Redis cache' do + worker.perform + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.get(described_class::LAST_PROCESSED_REDIS_KEY)).to be_nil + end + end + + it 'enqueues projects to be processed' do + worker.perform + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.lpop(described_class::QUEUE_KEY).to_i).to eq(project.id) + end + end + end + + context 'when the feature flag is disabled' do + before do + stub_feature_flags(ci_delete_old_pipelines: false) + end + + it 'does not enqueue DestroyOldPipelinesWorker jobs' do + expect(Ci::DestroyOldPipelinesWorker).not_to receive(:perform_with_capacity) + + worker.perform + end + + it 'does not enqueue projects to be processed' do + worker.perform + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.lpop(described_class::QUEUE_KEY)).to be_nil + end + end + end + end +end diff --git a/spec/workers/every_sidekiq_worker_spec.rb b/spec/workers/every_sidekiq_worker_spec.rb index 10e9823f308..192e51e8a47 100644 --- a/spec/workers/every_sidekiq_worker_spec.rb +++ b/spec/workers/every_sidekiq_worker_spec.rb @@ -482,7 +482,8 @@ RSpec.describe 'Every Sidekiq worker', feature_category: :shared do 'BulkImports::RelationBatchExportWorker' => 6, 'BulkImports::RelationExportWorker' => 6, 'Ci::Runners::ExportUsageCsvWorker' => 3, - 'AppSec::ContainerScanning::ScanImageWorker' => 3 + 'AppSec::ContainerScanning::ScanImageWorker' => 3, + 'Ci::DestroyOldPipelinesWorker' => 0 }.merge(extra_retry_exceptions) end