diff --git a/app/models/ci/pipeline_metadata.rb b/app/models/ci/pipeline_metadata.rb index 2bd206c5ca5..5457fb899b0 100644 --- a/app/models/ci/pipeline_metadata.rb +++ b/app/models/ci/pipeline_metadata.rb @@ -4,11 +4,17 @@ module Ci class PipelineMetadata < Ci::ApplicationRecord self.primary_key = :pipeline_id + enum auto_cancel_on_new_commit: { + conservative: 0, + interruptible: 1, + disabled: 2 + }, _prefix: true + belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :pipeline_metadata belongs_to :project, class_name: "Project", inverse_of: :pipeline_metadata validates :pipeline, presence: true validates :project, presence: true - validates :name, presence: true, length: { minimum: 1, maximum: 255 } + validates :name, length: { minimum: 1, maximum: 255 }, allow_nil: true end end diff --git a/app/models/concerns/cached_commit.rb b/app/models/concerns/cached_commit.rb index 8a53fec0612..74120f49d01 100644 --- a/app/models/concerns/cached_commit.rb +++ b/app/models/concerns/cached_commit.rb @@ -19,4 +19,8 @@ module CachedCommit def referenced_by [] end + + def extended_trailers + {} + end end diff --git a/app/models/merge_request_context_commit.rb b/app/models/merge_request_context_commit.rb index 281e11c7c13..921ad7e1f0a 100644 --- a/app/models/merge_request_context_commit.rb +++ b/app/models/merge_request_context_commit.rb @@ -26,6 +26,13 @@ class MergeRequestContextCommit < ApplicationRecord # create MergeRequestContextCommit by given commit sha and it's diff file record def self.bulk_insert(rows, **args) + # Remove the new extended_trailers attribute as this shouldn't be + # inserted into the database. This will be removed once the old + # format of the trailers attribute is deprecated. + rows = rows.map do |row| + row.except(:extended_trailers).to_hash + end + ApplicationRecord.legacy_bulk_insert('merge_request_context_commits', rows, **args) # rubocop:disable Gitlab/BulkInsert end diff --git a/app/models/merge_request_diff_commit.rb b/app/models/merge_request_diff_commit.rb index 790520c4123..d0d9f173346 100644 --- a/app/models/merge_request_diff_commit.rb +++ b/app/models/merge_request_diff_commit.rb @@ -53,8 +53,13 @@ class MergeRequestDiffCommit < ApplicationRecord # These fields are only used to determine the author/committer IDs, we # don't store them in the DB. + # + # Trailers are stored in the DB here in order to allow changelog parsing. + # Rather than add an additional column for :extended_trailers, we're instead + # ignoring it for now until we deprecate the :trailers field and replace it with + # the new functionality. commit_hash = commit_hash - .except(:author_name, :author_email, :committer_name, :committer_email) + .except(:author_name, :author_email, :committer_name, :committer_email, :extended_trailers) commit_hash.merge( commit_author_id: author.id, diff --git a/app/models/vulnerability.rb b/app/models/vulnerability.rb index 6bf63bab82c..c8f9e75a389 100644 --- a/app/models/vulnerability.rb +++ b/app/models/vulnerability.rb @@ -5,8 +5,8 @@ class Vulnerability < ApplicationRecord include EachBatch include IgnorableColumns - ignore_column %i[due_date epic_id milestone_id last_edited_at last_edited_by_id - start_date start_date_sourcing_milestone_id updated_by_id], + ignore_column %i[due_date due_date_sourcing_milestone_id epic_id milestone_id + last_edited_at last_edited_by_id start_date start_date_sourcing_milestone_id updated_by_id], remove_with: '16.9', remove_after: '2024-01-19' diff --git a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb index 316d30d94da..e2808f45821 100644 --- a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb +++ b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb @@ -8,6 +8,8 @@ module Gitlab extend ActiveSupport::Concern include JobDelayCalculator + attr_reader :project + ENQUEUED_JOB_COUNT = 'github-importer/enqueued_job_count/%{project}/%{collection}' included do @@ -17,8 +19,10 @@ module Gitlab # project_id - The ID of the GitLab project to import the note into. # hash - A Hash containing the details of the GitHub object to import. # notify_key - The Redis key to notify upon completion, if any. + def perform(project_id, hash, notify_key = nil) - project = Project.find_by_id(project_id) + @project = Project.find_by_id(project_id) # rubocop:disable Gitlab/ModuleWithInstanceVariables -- GitHub Import + # uses modules everywhere. Too big to refactor. return notify_waiter(notify_key) unless project diff --git a/config/events/k8s_api_proxy_requests_unique_users_via_ci_access.yml b/config/events/k8s_api_proxy_requests_unique_users_via_ci_access.yml new file mode 100644 index 00000000000..e1c7dc1bb28 --- /dev/null +++ b/config/events/k8s_api_proxy_requests_unique_users_via_ci_access.yml @@ -0,0 +1,22 @@ +--- +description: Kubernetes API proxy request using the CI/CD Tunnel via CI Access +category: InternalEventTracking +action: k8s_api_proxy_requests_unique_users_via_ci_access +label_description: +property_description: +value_description: +extra_properties: +identifiers: +- user +product_section: ops +product_stage: deploy +product_group: environments +milestone: "16.7" +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137488 +distributions: +- ce +- ee +tiers: +- free +- premium +- ultimate diff --git a/config/events/k8s_api_proxy_requests_unique_users_via_pat_access.yml b/config/events/k8s_api_proxy_requests_unique_users_via_pat_access.yml new file mode 100644 index 00000000000..d9b19bbfc1d --- /dev/null +++ b/config/events/k8s_api_proxy_requests_unique_users_via_pat_access.yml @@ -0,0 +1,22 @@ +--- +description: Kubernetes API proxy request via Personal Access Token Access +category: InternalEventTracking +action: k8s_api_proxy_requests_unique_users_via_pat_access +label_description: +property_description: +value_description: +extra_properties: +identifiers: +- user +product_section: ops +product_stage: deploy +product_group: environments +milestone: "16.7" +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137488 +distributions: +- ce +- ee +tiers: +- free +- premium +- ultimate diff --git a/config/events/k8s_api_proxy_requests_unique_users_via_user_access.yml b/config/events/k8s_api_proxy_requests_unique_users_via_user_access.yml new file mode 100644 index 00000000000..4dc5d4f7d4b --- /dev/null +++ b/config/events/k8s_api_proxy_requests_unique_users_via_user_access.yml @@ -0,0 +1,22 @@ +--- +description: Kubernetes API proxy request using the CI/CD Tunnel via User Access +category: InternalEventTracking +action: k8s_api_proxy_requests_unique_users_via_user_access +label_description: +property_description: +value_description: +extra_properties: +identifiers: +- user +product_section: ops +product_stage: deploy +product_group: environments +milestone: "16.7" +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137488 +distributions: +- ce +- ee +tiers: +- free +- premium +- ultimate diff --git a/config/feature_flags/development/github_import_increased_concurrent_workers.yml b/config/feature_flags/development/github_import_increased_concurrent_workers.yml new file mode 100644 index 00000000000..7e5adaadf62 --- /dev/null +++ b/config/feature_flags/development/github_import_increased_concurrent_workers.yml @@ -0,0 +1,8 @@ +--- +name: github_import_increased_concurrent_workers +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137832 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/432891 +milestone: '16.7' +type: development +group: group::import and integrate +default_enabled: false diff --git a/config/feature_flags/development/saml_microsoft_attribute_names.yml b/config/feature_flags/development/saml_microsoft_attribute_names.yml new file mode 100644 index 00000000000..808f6f1e3ab --- /dev/null +++ b/config/feature_flags/development/saml_microsoft_attribute_names.yml @@ -0,0 +1,8 @@ +--- +name: saml_microsoft_attribute_names +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/136734 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/430348 +milestone: '16.7' +type: development +group: group::authentication +default_enabled: false diff --git a/config/feature_flags/development/search_index_all_projects.yml b/config/feature_flags/development/search_index_all_projects.yml new file mode 100644 index 00000000000..dc4ea2a175d --- /dev/null +++ b/config/feature_flags/development/search_index_all_projects.yml @@ -0,0 +1,8 @@ +--- +name: search_index_all_projects +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134456 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/432489 +milestone: '16.7' +type: development +group: group::global search +default_enabled: false diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index e38c82e0a90..ef1199fefef 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -876,6 +876,10 @@ Gitlab.ee do Settings.cron_jobs['timeout_pending_status_check_responses_worker'] ||= {} Settings.cron_jobs['timeout_pending_status_check_responses_worker']['cron'] ||= '*/1 * * * *' Settings.cron_jobs['timeout_pending_status_check_responses_worker']['job_class'] = 'ComplianceManagement::TimeoutPendingStatusCheckResponsesWorker' + Settings.cron_jobs['click_house_ci_finished_builds_sync_worker'] ||= {} + Settings.cron_jobs['click_house_ci_finished_builds_sync_worker']['cron'] ||= '*/3 * * * *' + Settings.cron_jobs['click_house_ci_finished_builds_sync_worker']['args'] ||= [1] + Settings.cron_jobs['click_house_ci_finished_builds_sync_worker']['job_class'] = 'ClickHouse::CiFinishedBuildsSyncCronWorker' Gitlab.com do Settings.cron_jobs['disable_legacy_open_source_license_for_inactive_projects'] ||= {} @@ -893,10 +897,6 @@ Gitlab.ee do Settings.cron_jobs['click_house_events_sync_worker'] ||= {} Settings.cron_jobs['click_house_events_sync_worker']['cron'] ||= "*/3 * * * *" Settings.cron_jobs['click_house_events_sync_worker']['job_class'] = 'ClickHouse::EventsSyncWorker' - Settings.cron_jobs['click_house_ci_finished_builds_sync_worker'] ||= {} - Settings.cron_jobs['click_house_ci_finished_builds_sync_worker']['cron'] ||= '*/3 * * * *' - Settings.cron_jobs['click_house_ci_finished_builds_sync_worker']['args'] ||= [1] - Settings.cron_jobs['click_house_ci_finished_builds_sync_worker']['job_class'] = 'ClickHouse::CiFinishedBuildsSyncCronWorker' Settings.cron_jobs['vertex_ai_refresh_access_token_worker'] ||= {} Settings.cron_jobs['vertex_ai_refresh_access_token_worker']['cron'] ||= '*/50 * * * *' Settings.cron_jobs['vertex_ai_refresh_access_token_worker']['job_class'] = 'Llm::VertexAiAccessTokenRefreshWorker' @@ -975,22 +975,10 @@ Settings.repositories.storages.each do |key, storage| Settings.repositories.storages[key] = Gitlab::GitalyClient::StorageSettings.new(storage) end -# -# The repository_downloads_path is used to remove outdated repository -# archives, if someone has it configured incorrectly, and it points -# to the path where repositories are stored this can cause some -# data-integrity issue. In this case, we sets it to the default -# repository_downloads_path value. -# -repositories_storages = Settings.repositories.storages.values -repository_downloads_path = Settings.gitlab['repository_downloads_path'].to_s.gsub(%r{/$}, '') -repository_downloads_full_path = File.expand_path(repository_downloads_path, Settings.gitlab['user_home']) +repository_downloads_path = Settings.gitlab['repository_downloads_path'].to_s.gsub(%r{/$}, '') -# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1255 -Gitlab::GitalyClient::StorageSettings.allow_disk_access do - if repository_downloads_path.blank? || repositories_storages.any? { |rs| [repository_downloads_path, repository_downloads_full_path].include?(rs.legacy_disk_path.gsub(%r{/$}, '')) } - Settings.gitlab['repository_downloads_path'] = File.join(Settings.shared['path'], 'cache/archive') - end +if repository_downloads_path.blank? + Settings.gitlab['repository_downloads_path'] = File.join(Settings.shared['path'], 'cache/archive') end # diff --git a/config/metrics/counts_28d/20210216180312_snippets.yml b/config/metrics/counts_28d/20210216180312_snippets.yml index 7592ab4b461..38099be75ac 100644 --- a/config/metrics/counts_28d/20210216180312_snippets.yml +++ b/config/metrics/counts_28d/20210216180312_snippets.yml @@ -17,5 +17,6 @@ tier: - free - premium - ultimate -performance_indicator_type: [] +performance_indicator_type: +- customer_health_score milestone: "<13.9" diff --git a/config/metrics/counts_28d/20210216180334_g_edit_by_sfe_monthly.yml b/config/metrics/counts_28d/20210216180334_g_edit_by_sfe_monthly.yml index b9aca413d7c..8d68bd81e2f 100644 --- a/config/metrics/counts_28d/20210216180334_g_edit_by_sfe_monthly.yml +++ b/config/metrics/counts_28d/20210216180334_g_edit_by_sfe_monthly.yml @@ -23,5 +23,6 @@ tier: - free - premium - ultimate -performance_indicator_type: [] +performance_indicator_type: +- customer_health_score milestone: "<13.9" diff --git a/config/metrics/counts_28d/20230217215050_ci_internal_pipelines.yml b/config/metrics/counts_28d/20230217215050_ci_internal_pipelines.yml index acc9ae681f3..37c610568ca 100644 --- a/config/metrics/counts_28d/20230217215050_ci_internal_pipelines.yml +++ b/config/metrics/counts_28d/20230217215050_ci_internal_pipelines.yml @@ -12,7 +12,8 @@ time_frame: 28d data_source: database data_category: operational instrumentation_class: CountCiInternalPipelinesMetric -performance_indicator_type: [] +performance_indicator_type: +- customer_health_score distribution: - ce - ee diff --git a/config/metrics/counts_28d/20230620070723_k8s_api_proxy_requests_unique_users_via_ci_access_monthly.yml b/config/metrics/counts_28d/20230620070723_k8s_api_proxy_requests_unique_users_via_ci_access_monthly.yml index 10e0c13e8e3..177bfdd75a7 100644 --- a/config/metrics/counts_28d/20230620070723_k8s_api_proxy_requests_unique_users_via_ci_access_monthly.yml +++ b/config/metrics/counts_28d/20230620070723_k8s_api_proxy_requests_unique_users_via_ci_access_monthly.yml @@ -9,12 +9,15 @@ status: active milestone: "16.2" introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124147 time_frame: 28d -data_source: redis_hll +data_source: internal_events data_category: optional instrumentation_class: RedisHLLMetric options: events: - k8s_api_proxy_requests_unique_users_via_ci_access +events: + - name: k8s_api_proxy_requests_unique_users_via_ci_access + unique: user.id performance_indicator_type: [] distribution: - ce diff --git a/config/metrics/counts_28d/20230620070723_k8s_api_proxy_requests_unique_users_via_user_access_monthly.yml b/config/metrics/counts_28d/20230620070723_k8s_api_proxy_requests_unique_users_via_user_access_monthly.yml index 0ffcd3e9c75..bcda6d21dcf 100644 --- a/config/metrics/counts_28d/20230620070723_k8s_api_proxy_requests_unique_users_via_user_access_monthly.yml +++ b/config/metrics/counts_28d/20230620070723_k8s_api_proxy_requests_unique_users_via_user_access_monthly.yml @@ -9,12 +9,15 @@ status: active milestone: "16.2" introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124147 time_frame: 28d -data_source: redis_hll +data_source: internal_events data_category: optional instrumentation_class: RedisHLLMetric options: events: - k8s_api_proxy_requests_unique_users_via_user_access +events: + - name: k8s_api_proxy_requests_unique_users_via_user_access + unique: user.id performance_indicator_type: [] distribution: - ce diff --git a/config/metrics/counts_28d/20230816085153_k8s_api_proxy_requests_unique_users_via_pat_access_monthly.yml b/config/metrics/counts_28d/20230816085153_k8s_api_proxy_requests_unique_users_via_pat_access_monthly.yml index 5f987f91425..faee9fee9e8 100644 --- a/config/metrics/counts_28d/20230816085153_k8s_api_proxy_requests_unique_users_via_pat_access_monthly.yml +++ b/config/metrics/counts_28d/20230816085153_k8s_api_proxy_requests_unique_users_via_pat_access_monthly.yml @@ -9,12 +9,15 @@ status: active milestone: "16.4" introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/129463 time_frame: 28d -data_source: redis_hll +data_source: internal_events data_category: optional instrumentation_class: RedisHLLMetric options: events: - k8s_api_proxy_requests_unique_users_via_pat_access +events: + - name: k8s_api_proxy_requests_unique_users_via_pat_access + unique: user.id performance_indicator_type: [] distribution: - ce diff --git a/config/metrics/counts_7d/20230620070722_k8s_api_proxy_requests_unique_users_via_ci_access_weekly.yml b/config/metrics/counts_7d/20230620070722_k8s_api_proxy_requests_unique_users_via_ci_access_weekly.yml index 22580c0cf4a..52e486e551c 100644 --- a/config/metrics/counts_7d/20230620070722_k8s_api_proxy_requests_unique_users_via_ci_access_weekly.yml +++ b/config/metrics/counts_7d/20230620070722_k8s_api_proxy_requests_unique_users_via_ci_access_weekly.yml @@ -9,12 +9,15 @@ status: active milestone: "16.2" introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124147 time_frame: 7d -data_source: redis_hll +data_source: internal_events data_category: optional instrumentation_class: RedisHLLMetric options: events: - k8s_api_proxy_requests_unique_users_via_ci_access +events: + - name: k8s_api_proxy_requests_unique_users_via_ci_access + unique: user.id performance_indicator_type: [] distribution: - ce diff --git a/config/metrics/counts_7d/20230620070723_k8s_api_proxy_requests_unique_users_via_user_access_weekly.yml b/config/metrics/counts_7d/20230620070723_k8s_api_proxy_requests_unique_users_via_user_access_weekly.yml index 35d1fe7346b..9a60fe9f54d 100644 --- a/config/metrics/counts_7d/20230620070723_k8s_api_proxy_requests_unique_users_via_user_access_weekly.yml +++ b/config/metrics/counts_7d/20230620070723_k8s_api_proxy_requests_unique_users_via_user_access_weekly.yml @@ -9,12 +9,15 @@ status: active milestone: "16.2" introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124147 time_frame: 7d -data_source: redis_hll +data_source: internal_events data_category: optional instrumentation_class: RedisHLLMetric options: events: - k8s_api_proxy_requests_unique_users_via_user_access +events: + - name: k8s_api_proxy_requests_unique_users_via_user_access + unique: user.id performance_indicator_type: [] distribution: - ce diff --git a/config/metrics/counts_7d/20230816085153_k8s_api_proxy_requests_unique_users_via_pat_access_weekly.yml b/config/metrics/counts_7d/20230816085153_k8s_api_proxy_requests_unique_users_via_pat_access_weekly.yml index 5f3d0dc8912..96a07104165 100644 --- a/config/metrics/counts_7d/20230816085153_k8s_api_proxy_requests_unique_users_via_pat_access_weekly.yml +++ b/config/metrics/counts_7d/20230816085153_k8s_api_proxy_requests_unique_users_via_pat_access_weekly.yml @@ -9,12 +9,15 @@ status: active milestone: "16.4" introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/129463 time_frame: 7d -data_source: redis_hll +data_source: internal_events data_category: optional instrumentation_class: RedisHLLMetric options: events: - k8s_api_proxy_requests_unique_users_via_pat_access +events: + - name: k8s_api_proxy_requests_unique_users_via_pat_access + unique: user.id performance_indicator_type: [] distribution: - ce diff --git a/config/metrics/counts_all/20210216182006_source_code_pushes.yml b/config/metrics/counts_all/20210216182006_source_code_pushes.yml index a6b46fb44a8..a3ad620b8d2 100644 --- a/config/metrics/counts_all/20210216182006_source_code_pushes.yml +++ b/config/metrics/counts_all/20210216182006_source_code_pushes.yml @@ -22,5 +22,6 @@ tier: - free - premium - ultimate -performance_indicator_type: [] +performance_indicator_type: +- customer_health_score milestone: "<13.9" diff --git a/db/docs/projects.yml b/db/docs/projects.yml index 84c0aa3373f..36dc32be0ea 100644 --- a/db/docs/projects.yml +++ b/db/docs/projects.yml @@ -8,3 +8,22 @@ description: Stores project records introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/9ba1224867665844b117fa037e1465bb706b3685 milestone: "<6.0" gitlab_schema: gitlab_main_cell +schema_inconsistencies: +- type: missing_indexes + object_name: index_service_desk_enabled_projects_on_id_creator_id_created_at + introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137884 +- type: missing_indexes + object_name: index_projects_on_mirror_id_where_mirror_and_trigger_builds + introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137884 +- type: missing_indexes + object_name: index_projects_on_mirror_creator_id_created_at + introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137884 +- type: missing_indexes + object_name: index_projects_on_id_service_desk_enabled + introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137884 +- type: missing_indexes + object_name: idx_projects_id_created_at_disable_overriding_approvers_true + introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137884 +- type: missing_indexes + object_name: idx_projects_id_created_at_disable_overriding_approvers_false + introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137884 diff --git a/db/migrate/20231121092109_remove_ci_pipeline_metadata_name_not_null_constraint.rb b/db/migrate/20231121092109_remove_ci_pipeline_metadata_name_not_null_constraint.rb new file mode 100644 index 00000000000..de1d4c1bcc9 --- /dev/null +++ b/db/migrate/20231121092109_remove_ci_pipeline_metadata_name_not_null_constraint.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class RemoveCiPipelineMetadataNameNotNullConstraint < Gitlab::Database::Migration[2.2] + milestone '16.7' + + disable_ddl_transaction! + + CONSTRAINT_NAME = 'check_25d23931f1' + + def up + remove_not_null_constraint :ci_pipeline_metadata, :name, constraint_name: CONSTRAINT_NAME + end + + def down + add_not_null_constraint :ci_pipeline_metadata, :name, constraint_name: CONSTRAINT_NAME + end +end diff --git a/db/migrate/20231121092128_add_auto_cancel_on_new_commit_to_ci_pipeline_metadata.rb b/db/migrate/20231121092128_add_auto_cancel_on_new_commit_to_ci_pipeline_metadata.rb new file mode 100644 index 00000000000..9b698c83553 --- /dev/null +++ b/db/migrate/20231121092128_add_auto_cancel_on_new_commit_to_ci_pipeline_metadata.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class AddAutoCancelOnNewCommitToCiPipelineMetadata < Gitlab::Database::Migration[2.2] + enable_lock_retries! + milestone '16.7' + + def change + add_column :ci_pipeline_metadata, :auto_cancel_on_new_commit, :smallint, default: 0, null: false + end +end diff --git a/db/post_migrate/20231124105726_drop_idx_service_desk_enabled_projects_on_id_creator_id_created_at_for_gitlab_com.rb b/db/post_migrate/20231124105726_drop_idx_service_desk_enabled_projects_on_id_creator_id_created_at_for_gitlab_com.rb new file mode 100644 index 00000000000..96adba81123 --- /dev/null +++ b/db/post_migrate/20231124105726_drop_idx_service_desk_enabled_projects_on_id_creator_id_created_at_for_gitlab_com.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class DropIdxServiceDeskEnabledProjectsOnIdCreatorIdCreatedAtForGitlabCom < Gitlab::Database::Migration[2.2] + milestone '16.7' + + disable_ddl_transaction! + + TABLE_NAME = :projects + INDEX_NAME = :index_service_desk_enabled_projects_on_id_creator_id_created_at + + def up + return unless should_run? + + remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME + end + + def down + return unless should_run? + + add_concurrent_index( + TABLE_NAME, + [:id, :creator_id, :created_at], + where: 'service_desk_enabled = TRUE', + name: INDEX_NAME + ) + end + + private + + def should_run? + Gitlab.com_except_jh? + end +end diff --git a/db/post_migrate/20231124124600_drop_idx_projects_mirror_id_where_mirror_and_trigger_builds_for_gitlab_com.rb b/db/post_migrate/20231124124600_drop_idx_projects_mirror_id_where_mirror_and_trigger_builds_for_gitlab_com.rb new file mode 100644 index 00000000000..8e35a610361 --- /dev/null +++ b/db/post_migrate/20231124124600_drop_idx_projects_mirror_id_where_mirror_and_trigger_builds_for_gitlab_com.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class DropIdxProjectsMirrorIdWhereMirrorAndTriggerBuildsForGitlabCom < Gitlab::Database::Migration[2.2] + milestone '16.7' + + disable_ddl_transaction! + + TABLE_NAME = :projects + INDEX_NAME = :index_projects_on_mirror_id_where_mirror_and_trigger_builds + + def up + return unless should_run? + + remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME + end + + def down + return unless should_run? + + add_concurrent_index TABLE_NAME, :id, where: 'mirror = TRUE AND mirror_trigger_builds = TRUE', name: INDEX_NAME + end + + private + + def should_run? + Gitlab.com_except_jh? + end +end diff --git a/db/post_migrate/20231124124750_drop_idx_projects_on_mirror_creator_id_created_at_for_gitlab_com.rb b/db/post_migrate/20231124124750_drop_idx_projects_on_mirror_creator_id_created_at_for_gitlab_com.rb new file mode 100644 index 00000000000..2ea4967a154 --- /dev/null +++ b/db/post_migrate/20231124124750_drop_idx_projects_on_mirror_creator_id_created_at_for_gitlab_com.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class DropIdxProjectsOnMirrorCreatorIdCreatedAtForGitlabCom < Gitlab::Database::Migration[2.2] + milestone '16.7' + + disable_ddl_transaction! + + TABLE_NAME = :projects + INDEX_NAME = :index_projects_on_mirror_creator_id_created_at + + def up + return unless should_run? + + remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME + end + + def down + return unless should_run? + + add_concurrent_index( + TABLE_NAME, + [:creator_id, :created_at], + where: 'mirror = true and mirror_trigger_builds = true', + name: INDEX_NAME + ) + end + + private + + def should_run? + Gitlab.com_except_jh? + end +end diff --git a/db/post_migrate/20231124125007_drop_idx_projects_on_id_service_desk_enabled_for_gitlab_com.rb b/db/post_migrate/20231124125007_drop_idx_projects_on_id_service_desk_enabled_for_gitlab_com.rb new file mode 100644 index 00000000000..1bcdf18ae78 --- /dev/null +++ b/db/post_migrate/20231124125007_drop_idx_projects_on_id_service_desk_enabled_for_gitlab_com.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class DropIdxProjectsOnIdServiceDeskEnabledForGitlabCom < Gitlab::Database::Migration[2.2] + milestone '16.7' + + disable_ddl_transaction! + + TABLE_NAME = :projects + INDEX_NAME = :index_projects_on_id_service_desk_enabled + + def up + return unless should_run? + + remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME + end + + def down + return unless should_run? + + add_concurrent_index TABLE_NAME, :id, where: 'service_desk_enabled = true', name: INDEX_NAME + end + + private + + def should_run? + Gitlab.com_except_jh? + end +end diff --git a/db/post_migrate/20231124132953_drop_idx_projects_id_created_at_disable_overriding_approvers_true_for_gitlab_com.rb b/db/post_migrate/20231124132953_drop_idx_projects_id_created_at_disable_overriding_approvers_true_for_gitlab_com.rb new file mode 100644 index 00000000000..1d63c3ddb64 --- /dev/null +++ b/db/post_migrate/20231124132953_drop_idx_projects_id_created_at_disable_overriding_approvers_true_for_gitlab_com.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class DropIdxProjectsIdCreatedAtDisableOverridingApproversTrueForGitlabCom < Gitlab::Database::Migration[2.2] + milestone '16.7' + + disable_ddl_transaction! + + TABLE_NAME = :projects + INDEX_NAME = :idx_projects_id_created_at_disable_overriding_approvers_true + + def up + return unless should_run? + + remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME + end + + def down + return unless should_run? + + add_concurrent_index( + TABLE_NAME, + [:id, :created_at], + where: "disable_overriding_approvers_per_merge_request = TRUE", + name: INDEX_NAME + ) + end + + private + + def should_run? + Gitlab.com_except_jh? + end +end diff --git a/db/post_migrate/20231124133015_drop_idx_projects_id_created_at_disable_overriding_approvers_false_for_gitlab_com.rb b/db/post_migrate/20231124133015_drop_idx_projects_id_created_at_disable_overriding_approvers_false_for_gitlab_com.rb new file mode 100644 index 00000000000..8da8d726ab8 --- /dev/null +++ b/db/post_migrate/20231124133015_drop_idx_projects_id_created_at_disable_overriding_approvers_false_for_gitlab_com.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class DropIdxProjectsIdCreatedAtDisableOverridingApproversFalseForGitlabCom < Gitlab::Database::Migration[2.2] + milestone '16.7' + + disable_ddl_transaction! + + TABLE_NAME = :projects + INDEX_NAME = :idx_projects_id_created_at_disable_overriding_approvers_false + + def up + return unless should_run? + + remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME + end + + def down + return unless should_run? + + add_concurrent_index( + TABLE_NAME, + [:id, :created_at], + where: "(disable_overriding_approvers_per_merge_request = FALSE) OR " \ + "(disable_overriding_approvers_per_merge_request IS NULL)", + name: INDEX_NAME + ) + end + + private + + def should_run? + Gitlab.com_except_jh? + end +end diff --git a/db/schema_migrations/20231121092109 b/db/schema_migrations/20231121092109 new file mode 100644 index 00000000000..94d937167f7 --- /dev/null +++ b/db/schema_migrations/20231121092109 @@ -0,0 +1 @@ +22f8ce9668370446060d834b4a1fc8fe45fb5497a8bd9fc0fa7a0dc674416d2d \ No newline at end of file diff --git a/db/schema_migrations/20231121092128 b/db/schema_migrations/20231121092128 new file mode 100644 index 00000000000..aabcab23245 --- /dev/null +++ b/db/schema_migrations/20231121092128 @@ -0,0 +1 @@ +cb2ecf9b5e917a422f2372edf088ee0568cd1ecfd5d39288b5c641cf1594ad11 \ No newline at end of file diff --git a/db/schema_migrations/20231124105726 b/db/schema_migrations/20231124105726 new file mode 100644 index 00000000000..850db11da5a --- /dev/null +++ b/db/schema_migrations/20231124105726 @@ -0,0 +1 @@ +3ff6a61da6b815fd593a8ca64e3a1b5855d22e8e42c559f0a12d7240217e5ee2 \ No newline at end of file diff --git a/db/schema_migrations/20231124124600 b/db/schema_migrations/20231124124600 new file mode 100644 index 00000000000..8c4643a600a --- /dev/null +++ b/db/schema_migrations/20231124124600 @@ -0,0 +1 @@ +2883bd987e5a4b648adb5b65fd03fd98f96abaeaa5b0917a6e64cc8567e02a5e \ No newline at end of file diff --git a/db/schema_migrations/20231124124750 b/db/schema_migrations/20231124124750 new file mode 100644 index 00000000000..62403ccd209 --- /dev/null +++ b/db/schema_migrations/20231124124750 @@ -0,0 +1 @@ +d6c1ea9699f1487d464d51db6ee8949d97eb203b4086338043e02c5934268907 \ No newline at end of file diff --git a/db/schema_migrations/20231124125007 b/db/schema_migrations/20231124125007 new file mode 100644 index 00000000000..7c79f441f5d --- /dev/null +++ b/db/schema_migrations/20231124125007 @@ -0,0 +1 @@ +c800617b375789020ae585b159391a8ce82c151c81b66a6b0944cebff872298d \ No newline at end of file diff --git a/db/schema_migrations/20231124132953 b/db/schema_migrations/20231124132953 new file mode 100644 index 00000000000..04ef524c81a --- /dev/null +++ b/db/schema_migrations/20231124132953 @@ -0,0 +1 @@ +040a3ce9544fb221964fabcb7095ed22763939329547c0bb4d06efb2b6287a46 \ No newline at end of file diff --git a/db/schema_migrations/20231124133015 b/db/schema_migrations/20231124133015 new file mode 100644 index 00000000000..adea8a37ffc --- /dev/null +++ b/db/schema_migrations/20231124133015 @@ -0,0 +1 @@ +452224fbe86d7b5e81431f203121006d275d2a81089f6de9f453cefe099ef506 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index c4cecdc5468..1a9016eb6b1 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -14284,7 +14284,7 @@ CREATE TABLE ci_pipeline_metadata ( project_id bigint NOT NULL, pipeline_id bigint NOT NULL, name text, - CONSTRAINT check_25d23931f1 CHECK ((name IS NOT NULL)), + auto_cancel_on_new_commit smallint DEFAULT 0 NOT NULL, CONSTRAINT check_9d3665463c CHECK ((char_length(name) <= 255)) ); diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index a6e7f720b76..4ec051596d8 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -278,6 +278,7 @@ praefect['configuration'] = { database: { # ... host: POSTGRESQL_HOST, + user: 'praefect', port: 5432, password: PRAEFECT_SQL_PASSWORD, dbname: 'praefect_production', diff --git a/doc/api/api_resources.md b/doc/api/api_resources.md index 76c91b00eb9..cc748357c6f 100644 --- a/doc/api/api_resources.md +++ b/doc/api/api_resources.md @@ -34,13 +34,13 @@ The following API resources are available in the project context: | [Conan distributions](packages/conan.md) | `/projects/:id/packages/conan` (also available standalone) | | [Debian distributions](packages/debian_project_distributions.md) | `/projects/:id/debian_distributions` (also available for groups) | | [Debian packages](packages/debian.md) | `/projects/:id/packages/debian` (also available for groups) | -| [Dependencies](dependencies.md) **(ULTIMATE ALL)** | `/projects/:id/dependencies` | +| [Dependencies](dependencies.md) | `/projects/:id/dependencies` | | [Deploy keys](deploy_keys.md) | `/projects/:id/deploy_keys` (also available standalone) | | [Deploy tokens](deploy_tokens.md) | `/projects/:id/deploy_tokens` (also available for groups and standalone) | | [Deployments](deployments.md) | `/projects/:id/deployments` | | [Discussions](discussions.md) (threaded comments) | `/projects/:id/issues/.../discussions`, `/projects/:id/snippets/.../discussions`, `/projects/:id/merge_requests/.../discussions`, `/projects/:id/commits/.../discussions` (also available for groups) | -| [Draft Notes](draft_notes.md) (comments) | `/projects/:id/merge_requests/.../draft_notes` -| [Emoji reactions](emoji_reactions.md) | `/projects/:id/issues/.../award_emoji`, `/projects/:id/merge_requests/.../award_emoji`, `/projects/:id/snippets/.../award_emoji` | +| [Draft Notes](draft_notes.md) (comments) | `/projects/:id/merge_requests/.../draft_notes` +| [Emoji reactions](emoji_reactions.md) | `/projects/:id/issues/.../award_emoji`, `/projects/:id/merge_requests/.../award_emoji`, `/projects/:id/snippets/.../award_emoji` | | [Environments](environments.md) | `/projects/:id/environments` | | [Error Tracking](error_tracking.md) | `/projects/:id/error_tracking/settings` | | [Events](events.md) | `/projects/:id/events` (also available for users and standalone) | @@ -56,14 +56,14 @@ The following API resources are available in the project context: | [Issue links](issue_links.md) | `/projects/:id/issues/.../links` | | [Issues Statistics](issues_statistics.md) | `/projects/:id/issues_statistics` (also available for groups and standalone) | | [Issues](issues.md) | `/projects/:id/issues` (also available for groups and standalone) | -| [Iterations](iterations.md) **(PREMIUM ALL)** | `/projects/:id/iterations` (also available for groups) | +| [Iterations](iterations.md) | `/projects/:id/iterations` (also available for groups) | | [Project CI/CD job token scope](project_job_token_scopes.md) | `/projects/:id/job_token_scope` | | [Jobs](jobs.md) | `/projects/:id/jobs`, `/projects/:id/pipelines/.../jobs` | | [Jobs Artifacts](job_artifacts.md) | `/projects/:id/jobs/:job_id/artifacts` | | [Labels](labels.md) | `/projects/:id/labels` | | [Maven repository](packages/maven.md) | `/projects/:id/packages/maven` (also available for groups and standalone) | | [Members](members.md) | `/projects/:id/members` (also available for groups) | -| [Merge request approvals](merge_request_approvals.md) **(PREMIUM ALL)** | `/projects/:id/approvals`, `/projects/:id/merge_requests/.../approvals` | +| [Merge request approvals](merge_request_approvals.md) | `/projects/:id/approvals`, `/projects/:id/merge_requests/.../approvals` | | [Merge requests](merge_requests.md) | `/projects/:id/merge_requests` (also available for groups and standalone) | | [Merge trains](merge_trains.md) | `/projects/:id/merge_trains` | | [Metadata](metadata.md) | `/metadata` | @@ -82,7 +82,7 @@ The following API resources are available in the project context: | [Project milestones](milestones.md) | `/projects/:id/milestones` | | [Project snippets](project_snippets.md) | `/projects/:id/snippets` | | [Project templates](project_templates.md) | `/projects/:id/templates` | -| [Project vulnerabilities](project_vulnerabilities.md) **(ULTIMATE ALL)** | `/projects/:id/vulnerabilities` | +| [Project vulnerabilities](project_vulnerabilities.md). | `/projects/:id/vulnerabilities` | | [Project wikis](wikis.md) | `/projects/:id/wikis` | | [Project-level variables](project_level_variables.md) | `/projects/:id/variables` | | [Projects](projects.md) including setting Webhooks | `/projects`, `/projects/:id/hooks` (also available for users) | @@ -103,10 +103,10 @@ The following API resources are available in the project context: | [Tags](tags.md) | `/projects/:id/repository/tags` | | [Terraform modules](packages/terraform-modules.md) | `/projects/:id/packages/terraform/modules` (also available standalone) | | [User-starred metrics dashboards](metrics_user_starred_dashboards.md ) | `/projects/:id/metrics/user_starred_dashboards` | -| [Visual Review discussions](visual_review_discussions.md) **(PREMIUM ALL)** | `/projects/:id/merge_requests/:merge_request_id/visual_review_discussions` | -| [Vulnerabilities](vulnerabilities.md) **(ULTIMATE ALL)** | `/vulnerabilities/:id` | -| [Vulnerability exports](vulnerability_exports.md) **(ULTIMATE ALL)** | `/projects/:id/vulnerability_exports` | -| [Vulnerability findings](vulnerability_findings.md) **(ULTIMATE ALL)** | `/projects/:id/vulnerability_findings` | +| [Visual Review discussions](visual_review_discussions.md) | `/projects/:id/merge_requests/:merge_request_id/visual_review_discussions` | +| [Vulnerabilities](vulnerabilities.md) | `/vulnerabilities/:id` | +| [Vulnerability exports](vulnerability_exports.md) | `/projects/:id/vulnerability_exports` | +| [Vulnerability findings](vulnerability_findings.md) | `/projects/:id/vulnerability_findings` | ## Group resources @@ -120,19 +120,19 @@ The following API resources are available in the group context: | [Debian distributions](packages/debian_group_distributions.md) | `/groups/:id/-/packages/debian` (also available for projects) | | [Deploy tokens](deploy_tokens.md) | `/groups/:id/deploy_tokens` (also available for projects and standalone) | | [Discussions](discussions.md) (comments and threads) | `/groups/:id/epics/.../discussions` (also available for projects) | -| [Epic issues](epic_issues.md) **(PREMIUM ALL)** | `/groups/:id/epics/.../issues` | -| [Epic links](epic_links.md) **(PREMIUM ALL)** | `/groups/:id/epics/.../epics` | -| [Epics](epics.md) **(PREMIUM ALL)** | `/groups/:id/epics` | +| [Epic issues](epic_issues.md) | `/groups/:id/epics/.../issues` | +| [Epic links](epic_links.md) | `/groups/:id/epics/.../epics` | +| [Epics](epics.md) | `/groups/:id/epics` | | [Groups](groups.md) | `/groups`, `/groups/.../subgroups` | | [Group badges](group_badges.md) | `/groups/:id/badges` | | [Group issue boards](group_boards.md) | `/groups/:id/boards` | -| [Group iterations](group_iterations.md) **(PREMIUM ALL)** | `/groups/:id/iterations` (also available for projects) | +| [Group iterations](group_iterations.md) | `/groups/:id/iterations` (also available for projects) | | [Group labels](group_labels.md) | `/groups/:id/labels` | | [Group-level variables](group_level_variables.md) | `/groups/:id/variables` | | [Group milestones](group_milestones.md) | `/groups/:id/milestones` | | [Group releases](group_releases.md) | `/groups/:id/releases` | -| [Group SSH certificates](group_ssh_certificates.md) **(PREMIUM ALL)** | `/groups/:id/ssh_certificates` | -| [Group wikis](group_wikis.md) **(PREMIUM ALL)** | `/groups/:id/wikis` | +| [Group SSH certificates](group_ssh_certificates.md) | `/groups/:id/ssh_certificates` | +| [Group wikis](group_wikis.md) | `/groups/:id/wikis` | | [Invitations](invitations.md) | `/groups/:id/invitations` (also available for projects) | | [Issues](issues.md) | `/groups/:id/issues` (also available for projects and standalone) | | [Issues Statistics](issues_statistics.md) | `/groups/:id/issues_statistics` (also available for projects and standalone) | @@ -151,31 +151,31 @@ The following API resources are available outside of project and group contexts | Resource | Available endpoints | |:---------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------| -| [Appearance](appearance.md) **(FREE SELF)** | `/application/appearance` | +| [Appearance](appearance.md) | `/application/appearance` | | [Applications](applications.md) | `/applications` | -| [Audit Events](audit_events.md) **(PREMIUM SELF)** | `/audit_events` | +| [Audit Events](audit_events.md) | `/audit_events` | | [Avatar](avatar.md) | `/avatar` | | [Broadcast messages](broadcast_messages.md) | `/broadcast_messages` | | [Code snippets](snippets.md) | `/snippets` | | [Code Suggestions](code_suggestions.md) | `/code_suggestions` | | [Custom attributes](custom_attributes.md) | `/users/:id/custom_attributes` (also available for groups and projects) | -| [Dependency list exports](dependency_list_export.md) **(ULTIMATE ALL)** | `/pipelines/:id/dependency_list_exports`, `/projects/:id/dependency_list_exports`, `/groups/:id/dependency_list_exports`, `/security/dependency_list_exports/:id`, `/security/dependency_list_exports/:id/download` | +| [Dependency list exports](dependency_list_export.md) | `/pipelines/:id/dependency_list_exports`, `/projects/:id/dependency_list_exports`, `/groups/:id/dependency_list_exports`, `/security/dependency_list_exports/:id`, `/security/dependency_list_exports/:id/download` | | [Deploy keys](deploy_keys.md) | `/deploy_keys` (also available for projects) | | [Deploy tokens](deploy_tokens.md) | `/deploy_tokens` (also available for projects and groups) | | [Events](events.md) | `/events`, `/users/:id/events` (also available for projects) | | [Feature flags](features.md) | `/features` | -| [Geo Nodes](geo_nodes.md) **(PREMIUM SELF)** | `/geo_nodes` | +| [Geo Nodes](geo_nodes.md) | `/geo_nodes` | | [Group Activity Analytics](group_activity_analytics.md) | `/analytics/group_activity/{issues_count}` | -| [Group repository storage moves](group_repository_storage_moves.md) **(PREMIUM SELF)** | `/group_repository_storage_moves` | +| [Group repository storage moves](group_repository_storage_moves.md) | `/group_repository_storage_moves` | | [Import repository from GitHub](import.md#import-repository-from-github) | `/import/github` | | [Import repository from Bitbucket Server](import.md#import-repository-from-bitbucket-server) | `/import/bitbucket_server` | -| [Instance clusters](instance_clusters.md) **(FREE SELF)** | `/admin/clusters` | -| [Instance-level CI/CD variables](instance_level_ci_variables.md) **(FREE SELF)** | `/admin/ci/variables` | +| [Instance clusters](instance_clusters.md) | `/admin/clusters` | +| [Instance-level CI/CD variables](instance_level_ci_variables.md) | `/admin/ci/variables` | | [Issues Statistics](issues_statistics.md) | `/issues_statistics` (also available for groups and projects) | | [Issues](issues.md) | `/issues` (also available for groups and projects) | | [Jobs](jobs.md) | `/job` | | [Keys](keys.md) | `/keys` | -| [License](license.md) **(FREE SELF)** | `/license` | +| [License](license.md) | `/license` | | [Markdown](markdown.md) | `/markdown` | | [Merge requests](merge_requests.md) | `/merge_requests` (also available for groups and projects) | | [Metrics dashboard annotations](metrics_dashboard_annotations.md) | `/environments/:id/metrics_dashboard/annotations`, `/clusters/:id/metrics_dashboard/annotations` | @@ -184,15 +184,15 @@ The following API resources are available outside of project and group contexts | [Pages domains](pages_domains.md) | `/pages/domains` (also available for projects) | | [Personal access tokens](personal_access_tokens.md) | `/personal_access_tokens` | | [Plan limits](plan_limits.md) | `/application/plan_limits` | -| [Project repository storage moves](project_repository_storage_moves.md) **(FREE SELF)** | `/project_repository_storage_moves` | +| [Project repository storage moves](project_repository_storage_moves.md) | `/project_repository_storage_moves` | | [Projects](projects.md) | `/users/:id/projects` (also available for projects) | | [Runners](runners.md) | `/runners` (also available for projects) | | [Search](search.md) | `/search` (also available for groups and projects) | | [Service Data](usage_data.md) | `/usage_data` (For GitLab instance [Administrator](../user/permissions.md) users only) | -| [Settings](settings.md) **(FREE SELF)** | `/application/settings` | -| [Sidekiq metrics](sidekiq_metrics.md) **(FREE SELF)** | `/sidekiq` | -| [Sidekiq queues administration](admin_sidekiq_queues.md) **(FREE SELF)** | `/admin/sidekiq/queues/:queue_name` | -| [Snippet repository storage moves](snippet_repository_storage_moves.md) **(FREE SELF)** | `/snippet_repository_storage_moves` | +| [Settings](settings.md) | `/application/settings` | +| [Sidekiq metrics](sidekiq_metrics.md) | `/sidekiq` | +| [Sidekiq queues administration](admin_sidekiq_queues.md) | `/admin/sidekiq/queues/:queue_name` | +| [Snippet repository storage moves](snippet_repository_storage_moves.md) | `/snippet_repository_storage_moves` | | [Statistics](statistics.md) | `/application/statistics` | | [Suggestions](suggestions.md) | `/suggestions` | | [System hooks](system_hooks.md) | `/hooks` | diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index bf071e9ae51..f1855fd0d0b 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -827,6 +827,7 @@ Use `detailed_merge_status` instead of `merge_status` to account for all potenti - `not_approved`: Approval is required before merge. - `not_open`: The merge request must be open before merge. - `policies_denied`: The merge request contains denied policies. + - `jira_association_missing`: The title or description must reference a Jira issue. ### Preparation steps diff --git a/doc/integration/saml.md b/doc/integration/saml.md index 0c07d203ab0..607f269ef79 100644 --- a/doc/integration/saml.md +++ b/doc/integration/saml.md @@ -716,12 +716,23 @@ your provider's support. ### Configure assertions -| Field | Supported default keys | -|-----------------|------------------------| -| Email (required)| `email`, `mail` | -| Full Name | `name` | -| First Name | `first_name`, `firstname`, `firstName` | -| Last Name | `last_name`, `lastname`, `lastName` | +> - Microsoft Azure/Entra ID attribute support [introduced on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/420766) in GitLab 16.7. +> - Microsoft Azure/Entra ID attribute support [introduced on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/136734) in GitLab 16.7 [with a flag](../administration/feature_flags.md) named `saml_microsoft_attribute_names`. Disabled by default, and available to GitLab.com administrators only. + +FLAG: +On self-managed GitLab, Microsoft Azure/Entra ID attributes are supported by default. +In the following table, these attributes begin with either `http://schemas.xmlsoap.org` +or `http://schemas.microsoft.com`. +On GitLab.com, Microsoft Azure/Entra ID attributes are introduced +[with a flag](../administration/feature_flags.md) named `saml_microsoft_attribute_names`. +On GitLab.com, this feature is unavailable but can be configured by GitLab.com administrators only. + +| Field | Supported default keys | +|-----------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Email (required)| `email`, `mail`, `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress`, `http://schemas.microsoft.com/ws/2008/06/identity/claims/emailaddress` | +| Full Name | `name`, `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name`, `http://schemas.microsoft.com/ws/2008/06/identity/claims/name` | +| First Name | `first_name`, `firstname`, `firstName`, `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname`, `http://schemas.microsoft.com/ws/2008/06/identity/claims/givenname` | +| Last Name | `last_name`, `lastname`, `lastName`, `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname`, `http://schemas.microsoft.com/ws/2008/06/identity/claims/surname` | See [`attribute_statements`](#map-saml-response-attribute-names) for: diff --git a/doc/update/background_migrations.md b/doc/update/background_migrations.md index 5a4b19016f8..db9cce3ddc2 100644 --- a/doc/update/background_migrations.md +++ b/doc/update/background_migrations.md @@ -96,6 +96,15 @@ To query the database directly for the status of batched background migrations: WHERE status <> 3; ``` +Alternatively, you can wrap the query with `gitlab-psql -c ""` to check the status of +batched background migrations: + +```shell +gitlab-psql -c "SELECT job_class_name, table_name, column_name, job_arguments FROM batched_background_migrations WHERE status <> 3;" +``` + +If the query returns zero rows, all batched background migrations are complete. + ### Enable or disable advanced features Batched background migrations provide feature flags that enable you to customize diff --git a/doc/user/group/saml_sso/example_saml_config.md b/doc/user/group/saml_sso/example_saml_config.md index 86ad2ba32d1..d2bd56a2666 100644 --- a/doc/user/group/saml_sso/example_saml_config.md +++ b/doc/user/group/saml_sso/example_saml_config.md @@ -41,9 +41,6 @@ This section has screenshots for the elements of Azure Active Directory configur ![Azure AD user claims](img/AzureAD-claims.png) -NOTE: -Attribute names starting with phrases such as `http://schemas.microsoft.com/ws/2008/06/identity/claims/` are not supported. - ### SCIM mapping Provisioning: diff --git a/doc/user/group/saml_sso/index.md b/doc/user/group/saml_sso/index.md index 6482298d31d..c6131415c59 100644 --- a/doc/user/group/saml_sso/index.md +++ b/doc/user/group/saml_sso/index.md @@ -192,9 +192,6 @@ Optionally, you can pass user information to GitLab as attributes in the SAML as For more information, see the [attributes available for self-managed GitLab instances](../../../integration/saml.md#configure-assertions). -NOTE: -Attribute names starting with phrases such as `http://schemas.microsoft.com/ws/2008/06/identity/claims/` are not supported. For more information on configuring required attribute names in the SAML identity provider's settings, see [example group SAML and SCIM configurations](../../../user/group/saml_sso/example_saml_config.md). - ### Use metadata To configure some identity providers, you need a GitLab metadata URL. diff --git a/lib/api/entities/commit.rb b/lib/api/entities/commit.rb index ab1f51289d7..99ae4b66f67 100644 --- a/lib/api/entities/commit.rb +++ b/lib/api/entities/commit.rb @@ -17,6 +17,10 @@ module API expose :committer_email, documentation: { type: 'string', example: 'jack@example.com' } expose :committed_date, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } expose :trailers, documentation: { type: 'object', example: '{ "Merged-By": "Jane Doe janedoe@gitlab.com" }' } + expose :extended_trailers, documentation: { + type: 'object', + example: '{ "Signed-off-by": ["John Doe ", "Jane Doe "] }' + } expose :web_url, documentation: { diff --git a/lib/api/helpers/kubernetes/agent_helpers.rb b/lib/api/helpers/kubernetes/agent_helpers.rb index 654317ca616..ff80270fc15 100644 --- a/lib/api/helpers/kubernetes/agent_helpers.rb +++ b/lib/api/helpers/kubernetes/agent_helpers.rb @@ -43,9 +43,9 @@ module API def increment_unique_events events = params[:unique_counters]&.slice( :agent_users_using_ci_tunnel, - :k8s_api_proxy_requests_unique_users_via_ci_access, :k8s_api_proxy_requests_unique_agents_via_ci_access, - :k8s_api_proxy_requests_unique_users_via_user_access, :k8s_api_proxy_requests_unique_agents_via_user_access, - :k8s_api_proxy_requests_unique_users_via_pat_access, :k8s_api_proxy_requests_unique_agents_via_pat_access, + :k8s_api_proxy_requests_unique_agents_via_ci_access, + :k8s_api_proxy_requests_unique_agents_via_user_access, + :k8s_api_proxy_requests_unique_agents_via_pat_access, :flux_git_push_notified_unique_projects ) @@ -54,6 +54,27 @@ module API end end + def track_unique_user_events + events = params[:unique_counters]&.slice( + :k8s_api_proxy_requests_unique_users_via_ci_access, + :k8s_api_proxy_requests_unique_users_via_user_access, + :k8s_api_proxy_requests_unique_users_via_pat_access + ) + return if events.blank? + + unique_user_ids = events.values.flatten.uniq + users = User.id_in(unique_user_ids).index_by(&:id) + + events.each do |event, user_ids| + user_ids.each do |user_id| + user = users[user_id] + next if user.nil? + + Gitlab::InternalEvents.track_event(event, user: user) + end + end + end + def increment_count_events events = params[:counters]&.slice( :gitops_sync, :k8s_api_proxy_request, :flux_git_push_notifications_total, diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index 9b1c16d76bc..c6538e17b88 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -140,6 +140,7 @@ module API post '/', feature_category: :deployment_management do increment_count_events increment_unique_events + track_unique_user_events no_content! rescue ArgumentError => e diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 7524d8b9f85..235c472d292 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -4,10 +4,39 @@ module Gitlab module Auth module Saml class Config + DEFAULT_NICKNAME_ATTRS = %w[username nickname].freeze + DEFAULT_NAME_ATTRS = %w[ + http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name + http://schemas.microsoft.com/ws/2008/06/identity/claims/name + ].freeze + DEFAULT_EMAIL_ATTRS = %w[ + http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress + http://schemas.microsoft.com/ws/2008/06/identity/claims/emailaddress + ].freeze + DEFAULT_FIRST_NAME_ATTRS = %w[ + http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname + http://schemas.microsoft.com/ws/2008/06/identity/claims/givenname + ].freeze + DEFAULT_LAST_NAME_ATTRS = %w[ + http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname + http://schemas.microsoft.com/ws/2008/06/identity/claims/surname + ].freeze + class << self def enabled? ::AuthHelper.saml_providers.any? end + + def default_attribute_statements + defaults = OmniAuth::Strategies::SAML.default_options[:attribute_statements].to_hash.deep_symbolize_keys + defaults[:nickname] = DEFAULT_NICKNAME_ATTRS.dup + defaults[:name].concat(DEFAULT_NAME_ATTRS) + defaults[:email].concat(DEFAULT_EMAIL_ATTRS) + defaults[:first_name].concat(DEFAULT_FIRST_NAME_ATTRS) + defaults[:last_name].concat(DEFAULT_LAST_NAME_ATTRS) + + defaults + end end DEFAULT_PROVIDER_NAME = 'saml' diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 1086ea45a7a..d899ed3ba25 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -28,7 +28,8 @@ module Gitlab SERIALIZE_KEYS = [ :id, :message, :parent_ids, :authored_date, :author_name, :author_email, - :committed_date, :committer_name, :committer_email, :trailers, :referenced_by + :committed_date, :committer_name, :committer_email, + :trailers, :extended_trailers, :referenced_by ].freeze attr_accessor(*SERIALIZE_KEYS) @@ -432,9 +433,17 @@ module Gitlab @committer_email = commit.committer.email.dup @parent_ids = Array(commit.parent_ids) @trailers = commit.trailers.to_h { |t| [t.key, t.value] } + @extended_trailers = parse_commit_trailers(commit.trailers) @referenced_by = Array(commit.referenced_by) end + # Turn the commit trailers into a hash of key: [value, value] arrays + def parse_commit_trailers(trailers) + trailers.each_with_object({}) do |trailer, hash| + (hash[trailer.key] ||= []) << trailer.value + end + end + # Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone # offset in author.timezone. If the latter isn't present, assume UTC. def init_date_from_gitaly(author) diff --git a/lib/gitlab/github_import/job_delay_calculator.rb b/lib/gitlab/github_import/job_delay_calculator.rb index 077a27df16c..50cad1aae19 100644 --- a/lib/gitlab/github_import/job_delay_calculator.rb +++ b/lib/gitlab/github_import/job_delay_calculator.rb @@ -7,7 +7,9 @@ module Gitlab module JobDelayCalculator # Default batch settings for parallel import (can be redefined in Importer/Worker classes) def parallel_import_batch - { size: 1000, delay: 1.minute } + batch_size = Feature.enabled?(:github_import_increased_concurrent_workers, project.creator) ? 5000 : 1000 + + { size: batch_size, delay: 1.minute } end private diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 81ad7a7f9e1..1835aef755f 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -29,6 +29,8 @@ module Gitlab { authorize_params: { gl_auth_type: 'login' } } + when ->(provider_name) { AuthHelper.saml_providers.include?(provider_name.to_sym) } + { attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements } else {} end @@ -61,7 +63,7 @@ module Gitlab provider_arguments.concat arguments provider_arguments << defaults unless defaults.empty? when Hash, GitlabSettings::Options - hash_arguments = arguments.deep_symbolize_keys.deep_merge(defaults) + hash_arguments = merge_hash_defaults_and_args(defaults, arguments) normalized = normalize_hash_arguments(hash_arguments) # A Hash from the configuration will be passed as is. @@ -80,6 +82,15 @@ module Gitlab provider_arguments end + def merge_hash_defaults_and_args(defaults, arguments) + return arguments.to_hash if defaults.empty? + + revert_merging = Gitlab::Utils.to_boolean(ENV['REVERT_OMNIAUTH_DEFAULT_MERGING']) + return arguments.to_hash.deep_symbolize_keys.deep_merge(defaults) if revert_merging + + defaults.deep_merge(arguments.deep_symbolize_keys) + end + def normalize_hash_arguments(args) args.deep_symbolize_keys! diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 185b49d4a68..b0444066722 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -66,7 +66,7 @@ module Gitlab rescue StandardError => e # Ignore any exceptions unless is dev or test env - # The application flow should not be blocked by erros in tracking + # The application flow should not be blocked by errors in tracking Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end diff --git a/qa/Gemfile b/qa/Gemfile index 4085bed215a..a4b1d5f6079 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -41,7 +41,7 @@ gem 'chemlab-library-www-gitlab-com', '~> 0.1', '>= 0.1.1' # dependencies for jenkins client gem 'nokogiri', '~> 1.15', '>= 1.15.5' -gem 'deprecation_toolkit', '~> 2.0.3', require: false +gem 'deprecation_toolkit', '~> 2.0.4', require: false gem 'factory_bot', '~> 6.3.0' diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 6a8712bdfbc..18afcfb2f61 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -75,7 +75,7 @@ GEM crass (1.0.6) debug_inspector (1.1.0) declarative (0.0.20) - deprecation_toolkit (2.0.3) + deprecation_toolkit (2.0.4) activesupport (>= 5.2) diff-lcs (1.3) domain_name (0.5.20190701) @@ -357,7 +357,7 @@ DEPENDENCIES chemlab (~> 0.11, >= 0.11.1) chemlab-library-www-gitlab-com (~> 0.1, >= 0.1.1) confiner (~> 0.4) - deprecation_toolkit (~> 2.0.3) + deprecation_toolkit (~> 2.0.4) factory_bot (~> 6.3.0) faker (~> 3.2, >= 3.2.2) faraday-retry (~> 2.2) diff --git a/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_large_project_spec.rb b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_large_project_spec.rb index 6ea2047fed7..8761f0958b1 100644 --- a/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_large_project_spec.rb +++ b/qa/qa/specs/features/api/1_manage/migration/gitlab_migration_large_project_spec.rb @@ -66,7 +66,7 @@ module QA let(:source_project) { source_group.projects(auto_paginate: true).find { |project| project.name == gitlab_source_project }.reload! } let(:source_branches) { source_project.repository_branches(auto_paginate: true).map { |b| b[:name] } } let(:source_commits) { source_project.commits(auto_paginate: true).map { |c| c[:id] } } - let(:source_labels) { source_project.labels(auto_paginate: true).map { |l| l.except(:id) } } + let(:source_labels) { source_project.labels(auto_paginate: true).map { |l| l.except(:id, :description_html) } } let(:source_milestones) { source_project.milestones(auto_paginate: true).map { |ms| ms.except(:id, :web_url, :project_id) } } let(:source_mrs) { fetch_mrs(source_project, source_api_client, transform_urls: true) } let(:source_issues) { fetch_issues(source_project, source_api_client, transform_urls: true) } @@ -88,7 +88,7 @@ module QA let(:imported_project) { imported_group.projects(auto_paginate: true).find { |project| project.name == gitlab_source_project }.reload! } let(:branches) { imported_project.repository_branches(auto_paginate: true).map { |b| b[:name] } } let(:commits) { imported_project.commits(auto_paginate: true).map { |c| c[:id] } } - let(:labels) { imported_project.labels(auto_paginate: true).map { |l| l.except(:id) } } + let(:labels) { imported_project.labels(auto_paginate: true).map { |l| l.except(:id, :description_html) } } let(:milestones) { imported_project.milestones(auto_paginate: true).map { |ms| ms.except(:id, :web_url, :project_id) } } let(:mrs) { fetch_mrs(imported_project, api_client) } let(:issues) { fetch_issues(imported_project, api_client) } diff --git a/spec/factories/gitaly/commit.rb b/spec/factories/gitaly/commit.rb index 4e8220e449a..ecf3e4e065e 100644 --- a/spec/factories/gitaly/commit.rb +++ b/spec/factories/gitaly/commit.rb @@ -16,5 +16,15 @@ FactoryBot.define do body { subject + "\nMy body" } author { association(:gitaly_commit_author) } committer { association(:gitaly_commit_author) } + + trailers do + trailers = body.lines.keep_if { |l| l =~ /.*: / }.map do |l| + key, value = *l.split(":").map(&:strip) + + Gitaly::CommitTrailer.new(key: key, value: value) + end + + Google::Protobuf::RepeatedField.new(:message, Gitaly::CommitTrailer, trailers) + end end end diff --git a/spec/lib/gitlab/auth/saml/config_spec.rb b/spec/lib/gitlab/auth/saml/config_spec.rb index 2ecc26f9b96..bb5446e8d6a 100644 --- a/spec/lib/gitlab/auth/saml/config_spec.rb +++ b/spec/lib/gitlab/auth/saml/config_spec.rb @@ -19,6 +19,41 @@ RSpec.describe Gitlab::Auth::Saml::Config do end end + describe '.default_attribute_statements' do + it 'includes upstream defaults, nickname and Microsoft values' do + expect(described_class.default_attribute_statements).to eq( + { + nickname: %w[username nickname], + name: [ + 'name', + 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name', + 'http://schemas.microsoft.com/ws/2008/06/identity/claims/name' + ], + email: [ + 'email', + 'mail', + 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress', + 'http://schemas.microsoft.com/ws/2008/06/identity/claims/emailaddress' + ], + first_name: [ + 'first_name', + 'firstname', + 'firstName', + 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname', + 'http://schemas.microsoft.com/ws/2008/06/identity/claims/givenname' + ], + last_name: [ + 'last_name', + 'lastname', + 'lastName', + 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname', + 'http://schemas.microsoft.com/ws/2008/06/identity/claims/surname' + ] + } + ) + end + end + describe '#external_groups' do let(:config_1) { described_class.new('saml1') } diff --git a/spec/lib/gitlab/git/commit_spec.rb b/spec/lib/gitlab/git/commit_spec.rb index d8d62ac9670..6c8634281ae 100644 --- a/spec/lib/gitlab/git/commit_spec.rb +++ b/spec/lib/gitlab/git/commit_spec.rb @@ -8,7 +8,6 @@ RSpec.describe Gitlab::Git::Commit, feature_category: :source_code_management do describe "Commit info from gitaly commit" do let(:subject) { (+"My commit").force_encoding('ASCII-8BIT') } - let(:body) { subject + (+"My body").force_encoding('ASCII-8BIT') } let(:body_size) { body.length } let(:gitaly_commit) { build(:gitaly_commit, subject: subject, body: body, body_size: body_size, tree_id: tree_id) } let(:id) { gitaly_commit.id } @@ -17,6 +16,17 @@ RSpec.describe Gitlab::Git::Commit, feature_category: :source_code_management do let(:author) { gitaly_commit.author } let(:commit) { described_class.new(repository, gitaly_commit) } + let(:body) do + body = +<<~BODY + Bleep bloop. + + Cc: John Doe + Cc: Jane Doe + BODY + + [subject, "\n", body].join.force_encoding("ASCII-8BIT") + end + it { expect(commit.short_id).to eq(id[0..10]) } it { expect(commit.id).to eq(id) } it { expect(commit.sha).to eq(id) } @@ -29,6 +39,18 @@ RSpec.describe Gitlab::Git::Commit, feature_category: :source_code_management do it { expect(commit.parent_ids).to eq(gitaly_commit.parent_ids) } it { expect(commit.tree_id).to eq(tree_id) } + it "parses the commit trailers" do + expect(commit.trailers).to eq( + { "Cc" => "Jane Doe " } + ) + end + + it "parses the extended commit trailers" do + expect(commit.extended_trailers).to eq( + { "Cc" => ["John Doe ", "Jane Doe "] } + ) + end + context 'non-UTC dates' do let(:seconds) { Time.now.to_i } @@ -773,6 +795,7 @@ RSpec.describe Gitlab::Git::Commit, feature_category: :source_code_management do message: "tree css fixes", parent_ids: ["874797c3a73b60d2187ed6e2fcabd289ff75171e"], trailers: {}, + extended_trailers: {}, referenced_by: [] } end diff --git a/spec/lib/gitlab/github_import/importer/collaborators_importer_spec.rb b/spec/lib/gitlab/github_import/importer/collaborators_importer_spec.rb index 6f602531d23..63cd3926f02 100644 --- a/spec/lib/gitlab/github_import/importer/collaborators_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/collaborators_importer_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Gitlab::GithubImport::Importer::CollaboratorsImporter, feature_ca subject(:importer) { described_class.new(project, client, parallel: parallel) } let(:parallel) { true } - let(:project) { instance_double(Project, id: 4, import_source: 'foo/bar', import_state: nil) } + let(:project) { build(:project, id: 4, import_source: 'foo/bar', import_state: nil) } let(:client) { instance_double(Gitlab::GithubImport::Client) } let(:github_collaborator) do diff --git a/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb b/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb index 7668451ad4e..bcd38e1e236 100644 --- a/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNoteImporter, :aggregate_fail let_it_be(:project) { create(:project, :repository) } let_it_be(:user) { create(:user) } - let(:client) { double(:client) } + let(:client) { instance_double(Gitlab::GithubImport::Client) } let(:discussion_id) { 'b0fa404393eeebb4e82becb8104f238812bb1fe6' } let(:created_at) { Time.new(2017, 1, 1, 12, 00).utc } let(:updated_at) { Time.new(2017, 1, 1, 12, 15).utc } diff --git a/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb b/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb index 4e8066ecb69..b3bb449c963 100644 --- a/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::Importer::DiffNotesImporter, feature_category: :importers do - let(:project) { double(:project, id: 4, import_source: 'foo/bar') } - let(:client) { double(:client) } + let(:project) { build(:project, id: 4, import_source: 'foo/bar') } + let(:client) { instance_double(Gitlab::GithubImport::Client) } let(:github_comment) do { diff --git a/spec/lib/gitlab/github_import/importer/issue_events_importer_spec.rb b/spec/lib/gitlab/github_import/importer/issue_events_importer_spec.rb index 9aba6a2b02c..fb1e0c3caf7 100644 --- a/spec/lib/gitlab/github_import/importer/issue_events_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/issue_events_importer_spec.rb @@ -5,7 +5,7 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::Importer::IssueEventsImporter, feature_category: :importers do subject(:importer) { described_class.new(project, client, parallel: parallel) } - let(:project) { instance_double(Project, id: 4, import_source: 'foo/bar') } + let(:project) { build(:project, id: 4, import_source: 'foo/bar') } let(:client) { instance_double(Gitlab::GithubImport::Client) } let(:parallel) { true } diff --git a/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb b/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb index 1bfdce04187..7b42d3e1325 100644 --- a/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/issues_importer_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::Importer::IssuesImporter, feature_category: :importers do - let(:project) { double(:project, id: 4, import_source: 'foo/bar') } - let(:client) { double(:client) } + let(:project) { build(:project, id: 4, import_source: 'foo/bar') } + let(:client) { instance_double(Gitlab::GithubImport::Client) } let(:created_at) { Time.new(2017, 1, 1, 12, 00) } let(:updated_at) { Time.new(2017, 1, 1, 12, 15) } diff --git a/spec/lib/gitlab/github_import/importer/notes_importer_spec.rb b/spec/lib/gitlab/github_import/importer/notes_importer_spec.rb index 8c93963f325..99114a9be77 100644 --- a/spec/lib/gitlab/github_import/importer/notes_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/notes_importer_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::Importer::NotesImporter, feature_category: :importers do - let(:project) { double(:project, id: 4, import_source: 'foo/bar') } - let(:client) { double(:client) } + let(:project) { build(:project, id: 4, import_source: 'foo/bar') } + let(:client) { instance_double(Gitlab::GithubImport::Client) } let(:github_comment) do { diff --git a/spec/lib/gitlab/github_import/importer/protected_branches_importer_spec.rb b/spec/lib/gitlab/github_import/importer/protected_branches_importer_spec.rb index 8e99585109b..e30f3f34de1 100644 --- a/spec/lib/gitlab/github_import/importer/protected_branches_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/protected_branches_importer_spec.rb @@ -5,8 +5,8 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchesImporter, feature_category: :importers do subject(:importer) { described_class.new(project, client, parallel: parallel) } - let(:project) { instance_double('Project', id: 4, import_source: 'foo/bar') } - let(:client) { instance_double('Gitlab::GithubImport::Client') } + let(:project) { build(:project, id: 4, import_source: 'foo/bar') } + let(:client) { instance_double(Gitlab::GithubImport::Client) } let(:parallel) { true } let(:branches) do @@ -112,7 +112,7 @@ RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchesImporter, featur end it 'imports each protected branch in sequence' do - protected_branch_importer = instance_double('Gitlab::GithubImport::Importer::ProtectedBranchImporter') + protected_branch_importer = instance_double(Gitlab::GithubImport::Importer::ProtectedBranchImporter) expect(Gitlab::GithubImport::Importer::ProtectedBranchImporter) .to receive(:new) diff --git a/spec/lib/gitlab/github_import/job_delay_calculator_spec.rb b/spec/lib/gitlab/github_import/job_delay_calculator_spec.rb new file mode 100644 index 00000000000..3ddf8136dcf --- /dev/null +++ b/spec/lib/gitlab/github_import/job_delay_calculator_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::JobDelayCalculator, feature_category: :importers do + let(:project) { build(:project) } + + let(:importer_class) do + Class.new do + attr_reader :project + + def initialize(project) + @project = project + end + + include Gitlab::GithubImport::JobDelayCalculator + end + end + + describe "#parallel_import_batch" do + subject { importer_class.new(project).parallel_import_batch } + + it { is_expected.to eq({ size: 5000, delay: 1.minute }) } + + context 'when `github_import_increased_concurrent_workers` feature flag is disabled' do + before do + stub_feature_flags(github_import_increased_concurrent_workers: false) + end + + it { is_expected.to eq({ size: 1000, delay: 1.minute }) } + end + end +end diff --git a/spec/lib/gitlab/omniauth_initializer_spec.rb b/spec/lib/gitlab/omniauth_initializer_spec.rb index 9b46b8eccc8..222a730a229 100644 --- a/spec/lib/gitlab/omniauth_initializer_spec.rb +++ b/spec/lib/gitlab/omniauth_initializer_spec.rb @@ -2,7 +2,9 @@ require 'spec_helper' -RSpec.describe Gitlab::OmniauthInitializer do +RSpec.describe Gitlab::OmniauthInitializer, feature_category: :system_access do + include LoginHelpers + let(:devise_config) { class_double(Devise) } subject(:initializer) { described_class.new(devise_config) } @@ -224,6 +226,119 @@ RSpec.describe Gitlab::OmniauthInitializer do subject.execute([shibboleth_config]) end + context 'when SAML providers are configured' do + it 'configures default args for a single SAML provider' do + stub_omniauth_config(providers: [{ name: 'saml', args: { idp_sso_service_url: 'https://saml.example.com' } }]) + + expect(devise_config).to receive(:omniauth).with( + :saml, + { + idp_sso_service_url: 'https://saml.example.com', + attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements + } + ) + + initializer.execute(Gitlab.config.omniauth.providers) + end + + context 'when configuration provides matching keys' do + before do + stub_omniauth_config( + providers: [ + { + name: 'saml', + args: { idp_sso_service_url: 'https://saml.example.com', attribute_statements: { email: ['custom_attr'] } } + } + ] + ) + end + + it 'merges arguments with user configuration preference' do + expect(devise_config).to receive(:omniauth).with( + :saml, + { + idp_sso_service_url: 'https://saml.example.com', + attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements + .merge({ email: ['custom_attr'] }) + } + ) + + initializer.execute(Gitlab.config.omniauth.providers) + end + + it 'merges arguments with defaults preference when REVERT_OMNIAUTH_DEFAULT_MERGING is true' do + stub_env('REVERT_OMNIAUTH_DEFAULT_MERGING', 'true') + + expect(devise_config).to receive(:omniauth).with( + :saml, + { + idp_sso_service_url: 'https://saml.example.com', + attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements + } + ) + + initializer.execute(Gitlab.config.omniauth.providers) + end + end + + it 'configures defaults args for multiple SAML providers' do + stub_omniauth_config( + providers: [ + { name: 'saml', args: { idp_sso_service_url: 'https://saml.example.com' } }, + { + name: 'saml2', + args: { strategy_class: 'OmniAuth::Strategies::SAML', idp_sso_service_url: 'https://saml2.example.com' } + } + ] + ) + + expect(devise_config).to receive(:omniauth).with( + :saml, + { + idp_sso_service_url: 'https://saml.example.com', + attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements + } + ) + expect(devise_config).to receive(:omniauth).with( + :saml2, + { + idp_sso_service_url: 'https://saml2.example.com', + strategy_class: OmniAuth::Strategies::SAML, + attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements + } + ) + + initializer.execute(Gitlab.config.omniauth.providers) + end + + it 'merges arguments with user configuration preference for custom SAML provider' do + stub_omniauth_config( + providers: [ + { + name: 'custom_saml', + args: { + strategy_class: 'OmniAuth::Strategies::SAML', + idp_sso_service_url: 'https://saml2.example.com', + attribute_statements: { email: ['custom_attr'] } + } + } + ] + ) + + expect(devise_config).to receive(:omniauth).with( + :custom_saml, + { + idp_sso_service_url: 'https://saml2.example.com', + strategy_class: OmniAuth::Strategies::SAML, + attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements + .merge({ email: ['custom_attr'] }) + } + ) + + initializer.execute(Gitlab.config.omniauth.providers) + end + end + it 'configures defaults for google_oauth2' do google_config = { 'name' => 'google_oauth2', diff --git a/spec/models/ci/pipeline_metadata_spec.rb b/spec/models/ci/pipeline_metadata_spec.rb index 977c90bcc2a..2db7a2c23e9 100644 --- a/spec/models/ci/pipeline_metadata_spec.rb +++ b/spec/models/ci/pipeline_metadata_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Ci::PipelineMetadata do +RSpec.describe Ci::PipelineMetadata, feature_category: :pipeline_composition do it { is_expected.to belong_to(:project) } it { is_expected.to belong_to(:pipeline) } @@ -10,5 +10,13 @@ RSpec.describe Ci::PipelineMetadata do it { is_expected.to validate_length_of(:name).is_at_least(1).is_at_most(255) } it { is_expected.to validate_presence_of(:project) } it { is_expected.to validate_presence_of(:pipeline) } + + it do + is_expected.to define_enum_for( + :auto_cancel_on_new_commit + ).with_values( + conservative: 0, interruptible: 1, disabled: 2 + ).with_prefix + end end end diff --git a/spec/requests/api/commits_spec.rb b/spec/requests/api/commits_spec.rb index 6a112918288..4ec5d195ff8 100644 --- a/spec/requests/api/commits_spec.rb +++ b/spec/requests/api/commits_spec.rb @@ -44,6 +44,30 @@ RSpec.describe API::Commits, feature_category: :source_code_management do expect(response).to include_limited_pagination_headers end + + describe "commit trailers" do + it "doesn't include the commit trailers by default" do + get api(route, current_user), params: { page: 2 } + + commit_with_trailers = json_response.find { |c| c["trailers"].any? } + + expect(commit_with_trailers).to be_nil + expect(json_response.first["trailers"]).to eq({}) + end + + it "does include the commit trailers when specified in the params" do + # Test repo commits with trailers are further down the list, so use a + # higher page number. + get api(route, current_user), params: { page: 2, trailers: true } + + commit_with_trailers = json_response.find { |c| c["trailers"].any? } + + expect(commit_with_trailers["trailers"]).to be_a(Hash) + expect(commit_with_trailers["extended_trailers"]).to be_a(Hash) + expect(commit_with_trailers["trailers"].size).to be > 0 + expect(commit_with_trailers["extended_trailers"].size).to be > 0 + end + end end context 'when unauthenticated', 'and project is public' do @@ -426,6 +450,10 @@ RSpec.describe API::Commits, feature_category: :source_code_management do expect(commit['trailers']).to eq( 'Signed-off-by' => 'Dmitriy Zaporozhets ' ) + + expect(commit['extended_trailers']).to eq( + 'Signed-off-by' => ['Dmitriy Zaporozhets '] + ) end end end diff --git a/spec/requests/api/internal/kubernetes_spec.rb b/spec/requests/api/internal/kubernetes_spec.rb index d9d7133c386..f16fdcb9cc7 100644 --- a/spec/requests/api/internal/kubernetes_spec.rb +++ b/spec/requests/api/internal/kubernetes_spec.rb @@ -122,15 +122,17 @@ RSpec.describe API::Internal::Kubernetes, feature_category: :deployment_manageme k8s_api_proxy_requests_via_user_access: 44, k8s_api_proxy_requests_via_pat_access: 45 } + users = create_list(:user, 3) + user_ids = users.map(&:id) << users[0].id unique_counters = { - agent_users_using_ci_tunnel: [10, 999, 777, 10], - k8s_api_proxy_requests_unique_users_via_ci_access: [10, 999, 777, 10], - k8s_api_proxy_requests_unique_agents_via_ci_access: [10, 999, 777, 10], - k8s_api_proxy_requests_unique_users_via_user_access: [10, 999, 777, 10], - k8s_api_proxy_requests_unique_agents_via_user_access: [10, 999, 777, 10], - k8s_api_proxy_requests_unique_users_via_pat_access: [10, 999, 777, 10], - k8s_api_proxy_requests_unique_agents_via_pat_access: [10, 999, 777, 10], - flux_git_push_notified_unique_projects: [10, 999, 777, 10] + agent_users_using_ci_tunnel: user_ids, + k8s_api_proxy_requests_unique_users_via_ci_access: user_ids, + k8s_api_proxy_requests_unique_agents_via_ci_access: user_ids, + k8s_api_proxy_requests_unique_users_via_user_access: user_ids, + k8s_api_proxy_requests_unique_agents_via_user_access: user_ids, + k8s_api_proxy_requests_unique_users_via_pat_access: user_ids, + k8s_api_proxy_requests_unique_agents_via_pat_access: user_ids, + flux_git_push_notified_unique_projects: user_ids } expected_counters = { kubernetes_agent_gitops_sync: request_count * counters[:gitops_sync],