root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2023-11-28 18:11:25 +00:00
parent b6d3467a44
commit 27269330ae
74 changed files with 827 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/models/ci/pipeline_metadata.rb
View File

@ -4,11 +4,17 @@ module Ci
class PipelineMetadata < Ci::ApplicationRecord
self.primary_key = :pipeline_id
enum auto_cancel_on_new_commit: {
conservative: 0,
interruptible: 1,
disabled: 2
}, _prefix: true
belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :pipeline_metadata
belongs_to :project, class_name: "Project", inverse_of: :pipeline_metadata
validates :pipeline, presence: true
validates :project, presence: true
validates :name, presence: true, length: { minimum: 1, maximum: 255 }
validates :name, length: { minimum: 1, maximum: 255 }, allow_nil: true
end
end

4
app/models/concerns/cached_commit.rb
View File

@ -19,4 +19,8 @@ module CachedCommit
def referenced_by
[]
end
def extended_trailers
{}
end
end

7
app/models/merge_request_context_commit.rb
View File

@ -26,6 +26,13 @@ class MergeRequestContextCommit < ApplicationRecord
# create MergeRequestContextCommit by given commit sha and it's diff file record
def self.bulk_insert(rows, **args)
# Remove the new extended_trailers attribute as this shouldn't be
# inserted into the database. This will be removed once the old
# format of the trailers attribute is deprecated.
rows = rows.map do |row|
row.except(:extended_trailers).to_hash
end
ApplicationRecord.legacy_bulk_insert('merge_request_context_commits', rows, **args) # rubocop:disable Gitlab/BulkInsert
end

7
app/models/merge_request_diff_commit.rb
View File

@ -53,8 +53,13 @@ class MergeRequestDiffCommit < ApplicationRecord
# These fields are only used to determine the author/committer IDs, we
# don't store them in the DB.
#
# Trailers are stored in the DB here in order to allow changelog parsing.
# Rather than add an additional column for :extended_trailers, we're instead
# ignoring it for now until we deprecate the :trailers field and replace it with
# the new functionality.
commit_hash = commit_hash
.except(:author_name, :author_email, :committer_name, :committer_email)
.except(:author_name, :author_email, :committer_name, :committer_email, :extended_trailers)
commit_hash.merge(
commit_author_id: author.id,

4
app/models/vulnerability.rb
View File

@ -5,8 +5,8 @@ class Vulnerability < ApplicationRecord
include EachBatch
include IgnorableColumns
ignore_column %i[due_date epic_id milestone_id last_edited_at last_edited_by_id
start_date start_date_sourcing_milestone_id updated_by_id],
ignore_column %i[due_date due_date_sourcing_milestone_id epic_id milestone_id
last_edited_at last_edited_by_id start_date start_date_sourcing_milestone_id updated_by_id],
remove_with: '16.9',
remove_after: '2024-01-19'

6
app/workers/concerns/gitlab/github_import/rescheduling_methods.rb
View File

@ -8,6 +8,8 @@ module Gitlab
extend ActiveSupport::Concern
include JobDelayCalculator
attr_reader :project
ENQUEUED_JOB_COUNT = 'github-importer/enqueued_job_count/%{project}/%{collection}'
included do
@ -17,8 +19,10 @@ module Gitlab
# project_id - The ID of the GitLab project to import the note into.
# hash - A Hash containing the details of the GitHub object to import.
# notify_key - The Redis key to notify upon completion, if any.
def perform(project_id, hash, notify_key = nil)
project = Project.find_by_id(project_id)
@project = Project.find_by_id(project_id) # rubocop:disable Gitlab/ModuleWithInstanceVariables -- GitHub Import
# uses modules everywhere. Too big to refactor.
return notify_waiter(notify_key) unless project

22
config/events/k8s_api_proxy_requests_unique_users_via_ci_access.yml Normal file
View File

@ -0,0 +1,22 @@
---
description: Kubernetes API proxy request using the CI/CD Tunnel via CI Access
category: InternalEventTracking
action: k8s_api_proxy_requests_unique_users_via_ci_access
label_description:
property_description:
value_description:
extra_properties:
identifiers:
- user
product_section: ops
product_stage: deploy
product_group: environments
milestone: "16.7"
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137488
distributions:
- ce
- ee
tiers:
- free
- premium
- ultimate

22
config/events/k8s_api_proxy_requests_unique_users_via_pat_access.yml Normal file
View File

@ -0,0 +1,22 @@
---
description: Kubernetes API proxy request via Personal Access Token Access
category: InternalEventTracking
action: k8s_api_proxy_requests_unique_users_via_pat_access
label_description:
property_description:
value_description:
extra_properties:
identifiers:
- user
product_section: ops
product_stage: deploy
product_group: environments
milestone: "16.7"
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137488
distributions:
- ce
- ee
tiers:
- free
- premium
- ultimate

22
config/events/k8s_api_proxy_requests_unique_users_via_user_access.yml Normal file
View File

@ -0,0 +1,22 @@
---
description: Kubernetes API proxy request using the CI/CD Tunnel via User Access
category: InternalEventTracking
action: k8s_api_proxy_requests_unique_users_via_user_access
label_description:
property_description:
value_description:
extra_properties:
identifiers:
- user
product_section: ops
product_stage: deploy
product_group: environments
milestone: "16.7"
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137488
distributions:
- ce
- ee
tiers:
- free
- premium
- ultimate

8
config/feature_flags/development/github_import_increased_concurrent_workers.yml Normal file
View File

@ -0,0 +1,8 @@
---
name: github_import_increased_concurrent_workers
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137832
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/432891
milestone: '16.7'
type: development
group: group::import and integrate
default_enabled: false

8
config/feature_flags/development/saml_microsoft_attribute_names.yml Normal file
View File

@ -0,0 +1,8 @@
---
name: saml_microsoft_attribute_names
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/136734
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/430348
milestone: '16.7'
type: development
group: group::authentication
default_enabled: false

8
config/feature_flags/development/search_index_all_projects.yml Normal file
View File

@ -0,0 +1,8 @@
---
name: search_index_all_projects
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134456
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/432489
milestone: '16.7'
type: development
group: group::global search
default_enabled: false

26
config/initializers/1_settings.rb
View File

@ -876,6 +876,10 @@ Gitlab.ee do
Settings.cron_jobs['timeout_pending_status_check_responses_worker'] ||= {}
Settings.cron_jobs['timeout_pending_status_check_responses_worker']['cron'] ||= '*/1 * * * *'
Settings.cron_jobs['timeout_pending_status_check_responses_worker']['job_class'] = 'ComplianceManagement::TimeoutPendingStatusCheckResponsesWorker'
Settings.cron_jobs['click_house_ci_finished_builds_sync_worker'] ||= {}
Settings.cron_jobs['click_house_ci_finished_builds_sync_worker']['cron'] ||= '*/3 * * * *'
Settings.cron_jobs['click_house_ci_finished_builds_sync_worker']['args'] ||= [1]
Settings.cron_jobs['click_house_ci_finished_builds_sync_worker']['job_class'] = 'ClickHouse::CiFinishedBuildsSyncCronWorker'
Gitlab.com do
Settings.cron_jobs['disable_legacy_open_source_license_for_inactive_projects'] ||= {}
@ -893,10 +897,6 @@ Gitlab.ee do
Settings.cron_jobs['click_house_events_sync_worker'] ||= {}
Settings.cron_jobs['click_house_events_sync_worker']['cron'] ||= "*/3 * * * *"
Settings.cron_jobs['click_house_events_sync_worker']['job_class'] = 'ClickHouse::EventsSyncWorker'
Settings.cron_jobs['click_house_ci_finished_builds_sync_worker'] ||= {}
Settings.cron_jobs['click_house_ci_finished_builds_sync_worker']['cron'] ||= '*/3 * * * *'
Settings.cron_jobs['click_house_ci_finished_builds_sync_worker']['args'] ||= [1]
Settings.cron_jobs['click_house_ci_finished_builds_sync_worker']['job_class'] = 'ClickHouse::CiFinishedBuildsSyncCronWorker'
Settings.cron_jobs['vertex_ai_refresh_access_token_worker'] ||= {}
Settings.cron_jobs['vertex_ai_refresh_access_token_worker']['cron'] ||= '*/50 * * * *'
Settings.cron_jobs['vertex_ai_refresh_access_token_worker']['job_class'] = 'Llm::VertexAiAccessTokenRefreshWorker'
@ -975,22 +975,10 @@ Settings.repositories.storages.each do |key, storage|
Settings.repositories.storages[key] = Gitlab::GitalyClient::StorageSettings.new(storage)
end
#
# The repository_downloads_path is used to remove outdated repository
# archives, if someone has it configured incorrectly, and it points
# to the path where repositories are stored this can cause some
# data-integrity issue. In this case, we sets it to the default
# repository_downloads_path value.
#
repositories_storages = Settings.repositories.storages.values
repository_downloads_path = Settings.gitlab['repository_downloads_path'].to_s.gsub(%r{/$}, '')
repository_downloads_full_path = File.expand_path(repository_downloads_path, Settings.gitlab['user_home'])
repository_downloads_path = Settings.gitlab['repository_downloads_path'].to_s.gsub(%r{/$}, '')
# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1255
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
if repository_downloads_path.blank? || repositories_storages.any? { |rs| [repository_downloads_path, repository_downloads_full_path].include?(rs.legacy_disk_path.gsub(%r{/$}, '')) }
Settings.gitlab['repository_downloads_path'] = File.join(Settings.shared['path'], 'cache/archive')
end
if repository_downloads_path.blank?
Settings.gitlab['repository_downloads_path'] = File.join(Settings.shared['path'], 'cache/archive')
end
#

3
config/metrics/counts_28d/20210216180312_snippets.yml
View File

@ -17,5 +17,6 @@ tier:
- free
- premium
- ultimate
performance_indicator_type: []
performance_indicator_type:
- customer_health_score
milestone: "<13.9"

3
config/metrics/counts_28d/20210216180334_g_edit_by_sfe_monthly.yml
View File

@ -23,5 +23,6 @@ tier:
- free
- premium
- ultimate
performance_indicator_type: []
performance_indicator_type:
- customer_health_score
milestone: "<13.9"

3
config/metrics/counts_28d/20230217215050_ci_internal_pipelines.yml
View File

@ -12,7 +12,8 @@ time_frame: 28d
data_source: database
data_category: operational
instrumentation_class: CountCiInternalPipelinesMetric
performance_indicator_type: []
performance_indicator_type:
- customer_health_score
distribution:
- ce
- ee

5
config/metrics/counts_28d/20230620070723_k8s_api_proxy_requests_unique_users_via_ci_access_monthly.yml
View File

@ -9,12 +9,15 @@ status: active
milestone: "16.2"
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124147
time_frame: 28d
data_source: redis_hll
data_source: internal_events
data_category: optional
instrumentation_class: RedisHLLMetric
options:
events:
- k8s_api_proxy_requests_unique_users_via_ci_access
events:
- name: k8s_api_proxy_requests_unique_users_via_ci_access
unique: user.id
performance_indicator_type: []
distribution:
- ce

5
config/metrics/counts_28d/20230620070723_k8s_api_proxy_requests_unique_users_via_user_access_monthly.yml
View File

@ -9,12 +9,15 @@ status: active
milestone: "16.2"
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124147
time_frame: 28d
data_source: redis_hll
data_source: internal_events
data_category: optional
instrumentation_class: RedisHLLMetric
options:
events:
- k8s_api_proxy_requests_unique_users_via_user_access
events:
- name: k8s_api_proxy_requests_unique_users_via_user_access
unique: user.id
performance_indicator_type: []
distribution:
- ce

5
config/metrics/counts_28d/20230816085153_k8s_api_proxy_requests_unique_users_via_pat_access_monthly.yml
View File

@ -9,12 +9,15 @@ status: active
milestone: "16.4"
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/129463
time_frame: 28d
data_source: redis_hll
data_source: internal_events
data_category: optional
instrumentation_class: RedisHLLMetric
options:
events:
- k8s_api_proxy_requests_unique_users_via_pat_access
events:
- name: k8s_api_proxy_requests_unique_users_via_pat_access
unique: user.id
performance_indicator_type: []
distribution:
- ce

5
config/metrics/counts_7d/20230620070722_k8s_api_proxy_requests_unique_users_via_ci_access_weekly.yml
View File

@ -9,12 +9,15 @@ status: active
milestone: "16.2"
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124147
time_frame: 7d
data_source: redis_hll
data_source: internal_events
data_category: optional
instrumentation_class: RedisHLLMetric
options:
events:
- k8s_api_proxy_requests_unique_users_via_ci_access
events:
- name: k8s_api_proxy_requests_unique_users_via_ci_access
unique: user.id
performance_indicator_type: []
distribution:
- ce

5
config/metrics/counts_7d/20230620070723_k8s_api_proxy_requests_unique_users_via_user_access_weekly.yml
View File

@ -9,12 +9,15 @@ status: active
milestone: "16.2"
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124147
time_frame: 7d
data_source: redis_hll
data_source: internal_events
data_category: optional
instrumentation_class: RedisHLLMetric
options:
events:
- k8s_api_proxy_requests_unique_users_via_user_access
events:
- name: k8s_api_proxy_requests_unique_users_via_user_access
unique: user.id
performance_indicator_type: []
distribution:
- ce

5
config/metrics/counts_7d/20230816085153_k8s_api_proxy_requests_unique_users_via_pat_access_weekly.yml
View File

@ -9,12 +9,15 @@ status: active
milestone: "16.4"
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/129463
time_frame: 7d
data_source: redis_hll
data_source: internal_events
data_category: optional
instrumentation_class: RedisHLLMetric
options:
events:
- k8s_api_proxy_requests_unique_users_via_pat_access
events:
- name: k8s_api_proxy_requests_unique_users_via_pat_access
unique: user.id
performance_indicator_type: []
distribution:
- ce

3
config/metrics/counts_all/20210216182006_source_code_pushes.yml
View File

@ -22,5 +22,6 @@ tier:
- free
- premium
- ultimate
performance_indicator_type: []
performance_indicator_type:
- customer_health_score
milestone: "<13.9"

19
db/docs/projects.yml
View File

@ -8,3 +8,22 @@ description: Stores project records
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/9ba1224867665844b117fa037e1465bb706b3685
milestone: "<6.0"
gitlab_schema: gitlab_main_cell
schema_inconsistencies:
- type: missing_indexes
object_name: index_service_desk_enabled_projects_on_id_creator_id_created_at
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137884
- type: missing_indexes
object_name: index_projects_on_mirror_id_where_mirror_and_trigger_builds
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137884
- type: missing_indexes
object_name: index_projects_on_mirror_creator_id_created_at
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137884
- type: missing_indexes
object_name: index_projects_on_id_service_desk_enabled
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137884
- type: missing_indexes
object_name: idx_projects_id_created_at_disable_overriding_approvers_true
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137884
- type: missing_indexes
object_name: idx_projects_id_created_at_disable_overriding_approvers_false
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137884

17
db/migrate/20231121092109_remove_ci_pipeline_metadata_name_not_null_constraint.rb Normal file
View File

@ -0,0 +1,17 @@
# frozen_string_literal: true
class RemoveCiPipelineMetadataNameNotNullConstraint < Gitlab::Database::Migration[2.2]
milestone '16.7'
disable_ddl_transaction!
CONSTRAINT_NAME = 'check_25d23931f1'
def up
remove_not_null_constraint :ci_pipeline_metadata, :name, constraint_name: CONSTRAINT_NAME
end
def down
add_not_null_constraint :ci_pipeline_metadata, :name, constraint_name: CONSTRAINT_NAME
end
end

10
db/migrate/20231121092128_add_auto_cancel_on_new_commit_to_ci_pipeline_metadata.rb Normal file
View File

@ -0,0 +1,10 @@
# frozen_string_literal: true
class AddAutoCancelOnNewCommitToCiPipelineMetadata < Gitlab::Database::Migration[2.2]
enable_lock_retries!
milestone '16.7'
def change
add_column :ci_pipeline_metadata, :auto_cancel_on_new_commit, :smallint, default: 0, null: false
end
end

33
db/post_migrate/20231124105726_drop_idx_service_desk_enabled_projects_on_id_creator_id_created_at_for_gitlab_com.rb Normal file
View File

@ -0,0 +1,33 @@
# frozen_string_literal: true
class DropIdxServiceDeskEnabledProjectsOnIdCreatorIdCreatedAtForGitlabCom < Gitlab::Database::Migration[2.2]
milestone '16.7'
disable_ddl_transaction!
TABLE_NAME = :projects
INDEX_NAME = :index_service_desk_enabled_projects_on_id_creator_id_created_at
def up
return unless should_run?
remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME
end
def down
return unless should_run?
add_concurrent_index(
TABLE_NAME,
[:id, :creator_id, :created_at],
where: 'service_desk_enabled = TRUE',
name: INDEX_NAME
)
end
private
def should_run?
Gitlab.com_except_jh?
end
end

28
db/post_migrate/20231124124600_drop_idx_projects_mirror_id_where_mirror_and_trigger_builds_for_gitlab_com.rb Normal file
View File

@ -0,0 +1,28 @@
# frozen_string_literal: true
class DropIdxProjectsMirrorIdWhereMirrorAndTriggerBuildsForGitlabCom < Gitlab::Database::Migration[2.2]
milestone '16.7'
disable_ddl_transaction!
TABLE_NAME = :projects
INDEX_NAME = :index_projects_on_mirror_id_where_mirror_and_trigger_builds
def up
return unless should_run?
remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME
end
def down
return unless should_run?
add_concurrent_index TABLE_NAME, :id, where: 'mirror = TRUE AND mirror_trigger_builds = TRUE', name: INDEX_NAME
end
private
def should_run?
Gitlab.com_except_jh?
end
end

33
db/post_migrate/20231124124750_drop_idx_projects_on_mirror_creator_id_created_at_for_gitlab_com.rb Normal file
View File

@ -0,0 +1,33 @@
# frozen_string_literal: true
class DropIdxProjectsOnMirrorCreatorIdCreatedAtForGitlabCom < Gitlab::Database::Migration[2.2]
milestone '16.7'
disable_ddl_transaction!
TABLE_NAME = :projects
INDEX_NAME = :index_projects_on_mirror_creator_id_created_at
def up
return unless should_run?
remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME
end
def down
return unless should_run?
add_concurrent_index(
TABLE_NAME,
[:creator_id, :created_at],
where: 'mirror = true and mirror_trigger_builds = true',
name: INDEX_NAME
)
end
private
def should_run?
Gitlab.com_except_jh?
end
end

28
db/post_migrate/20231124125007_drop_idx_projects_on_id_service_desk_enabled_for_gitlab_com.rb Normal file
View File

@ -0,0 +1,28 @@
# frozen_string_literal: true
class DropIdxProjectsOnIdServiceDeskEnabledForGitlabCom < Gitlab::Database::Migration[2.2]
milestone '16.7'
disable_ddl_transaction!
TABLE_NAME = :projects
INDEX_NAME = :index_projects_on_id_service_desk_enabled
def up
return unless should_run?
remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME
end
def down
return unless should_run?
add_concurrent_index TABLE_NAME, :id, where: 'service_desk_enabled = true', name: INDEX_NAME
end
private
def should_run?
Gitlab.com_except_jh?
end
end

33
db/post_migrate/20231124132953_drop_idx_projects_id_created_at_disable_overriding_approvers_true_for_gitlab_com.rb Normal file
View File

@ -0,0 +1,33 @@
# frozen_string_literal: true
class DropIdxProjectsIdCreatedAtDisableOverridingApproversTrueForGitlabCom < Gitlab::Database::Migration[2.2]
milestone '16.7'
disable_ddl_transaction!
TABLE_NAME = :projects
INDEX_NAME = :idx_projects_id_created_at_disable_overriding_approvers_true
def up
return unless should_run?
remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME
end
def down
return unless should_run?
add_concurrent_index(
TABLE_NAME,
[:id, :created_at],
where: "disable_overriding_approvers_per_merge_request = TRUE",
name: INDEX_NAME
)
end
private
def should_run?
Gitlab.com_except_jh?
end
end

34
db/post_migrate/20231124133015_drop_idx_projects_id_created_at_disable_overriding_approvers_false_for_gitlab_com.rb Normal file
View File

@ -0,0 +1,34 @@
# frozen_string_literal: true
class DropIdxProjectsIdCreatedAtDisableOverridingApproversFalseForGitlabCom < Gitlab::Database::Migration[2.2]
milestone '16.7'
disable_ddl_transaction!
TABLE_NAME = :projects
INDEX_NAME = :idx_projects_id_created_at_disable_overriding_approvers_false
def up
return unless should_run?
remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME
end
def down
return unless should_run?
add_concurrent_index(
TABLE_NAME,
[:id, :created_at],
where: "(disable_overriding_approvers_per_merge_request = FALSE) OR " \
"(disable_overriding_approvers_per_merge_request IS NULL)",
name: INDEX_NAME
)
end
private
def should_run?
Gitlab.com_except_jh?
end
end

1
db/schema_migrations/20231121092109 Normal file
View File

@ -0,0 +1 @@
22f8ce9668370446060d834b4a1fc8fe45fb5497a8bd9fc0fa7a0dc674416d2d

1
db/schema_migrations/20231121092128 Normal file
View File

@ -0,0 +1 @@
cb2ecf9b5e917a422f2372edf088ee0568cd1ecfd5d39288b5c641cf1594ad11

1
db/schema_migrations/20231124105726 Normal file
View File

@ -0,0 +1 @@
3ff6a61da6b815fd593a8ca64e3a1b5855d22e8e42c559f0a12d7240217e5ee2

1
db/schema_migrations/20231124124600 Normal file
View File

@ -0,0 +1 @@
2883bd987e5a4b648adb5b65fd03fd98f96abaeaa5b0917a6e64cc8567e02a5e

1
db/schema_migrations/20231124124750 Normal file
View File

@ -0,0 +1 @@
d6c1ea9699f1487d464d51db6ee8949d97eb203b4086338043e02c5934268907

1
db/schema_migrations/20231124125007 Normal file
View File

@ -0,0 +1 @@
c800617b375789020ae585b159391a8ce82c151c81b66a6b0944cebff872298d

1
db/schema_migrations/20231124132953 Normal file
View File

@ -0,0 +1 @@
040a3ce9544fb221964fabcb7095ed22763939329547c0bb4d06efb2b6287a46

1
db/schema_migrations/20231124133015 Normal file
View File

@ -0,0 +1 @@
452224fbe86d7b5e81431f203121006d275d2a81089f6de9f453cefe099ef506

2
db/structure.sql
View File

@ -14284,7 +14284,7 @@ CREATE TABLE ci_pipeline_metadata (
project_id bigint NOT NULL,
pipeline_id bigint NOT NULL,
name text,
CONSTRAINT check_25d23931f1 CHECK ((name IS NOT NULL)),
auto_cancel_on_new_commit smallint DEFAULT 0 NOT NULL,
CONSTRAINT check_9d3665463c CHECK ((char_length(name) <= 255))
);

1
doc/administration/gitaly/praefect.md
View File

@ -278,6 +278,7 @@ praefect['configuration'] = {
database: {
# ...
host: POSTGRESQL_HOST,
user: 'praefect',
port: 5432,
password: PRAEFECT_SQL_PASSWORD,
dbname: 'praefect_production',

58
doc/api/api_resources.md
View File

@ -34,13 +34,13 @@ The following API resources are available in the project context:
| [Conan distributions](packages/conan.md) | `/projects/:id/packages/conan` (also available standalone) |
| [Debian distributions](packages/debian_project_distributions.md) | `/projects/:id/debian_distributions` (also available for groups) |
| [Debian packages](packages/debian.md) | `/projects/:id/packages/debian` (also available for groups) |
| [Dependencies](dependencies.md) **(ULTIMATE ALL)** | `/projects/:id/dependencies` |
| [Dependencies](dependencies.md) | `/projects/:id/dependencies` |
| [Deploy keys](deploy_keys.md) | `/projects/:id/deploy_keys` (also available standalone) |
| [Deploy tokens](deploy_tokens.md) | `/projects/:id/deploy_tokens` (also available for groups and standalone) |
| [Deployments](deployments.md) | `/projects/:id/deployments` |
| [Discussions](discussions.md) (threaded comments) | `/projects/:id/issues/.../discussions`, `/projects/:id/snippets/.../discussions`, `/projects/:id/merge_requests/.../discussions`, `/projects/:id/commits/.../discussions` (also available for groups) |
| [Draft Notes](draft_notes.md) (comments) | `/projects/:id/merge_requests/.../draft_notes`
| [Emoji reactions](emoji_reactions.md) | `/projects/:id/issues/.../award_emoji`, `/projects/:id/merge_requests/.../award_emoji`, `/projects/:id/snippets/.../award_emoji` |
| [Draft Notes](draft_notes.md) (comments) | `/projects/:id/merge_requests/.../draft_notes`
| [Emoji reactions](emoji_reactions.md) | `/projects/:id/issues/.../award_emoji`, `/projects/:id/merge_requests/.../award_emoji`, `/projects/:id/snippets/.../award_emoji` |
| [Environments](environments.md) | `/projects/:id/environments` |
| [Error Tracking](error_tracking.md) | `/projects/:id/error_tracking/settings` |
| [Events](events.md) | `/projects/:id/events` (also available for users and standalone) |
@ -56,14 +56,14 @@ The following API resources are available in the project context:
| [Issue links](issue_links.md) | `/projects/:id/issues/.../links` |
| [Issues Statistics](issues_statistics.md) | `/projects/:id/issues_statistics` (also available for groups and standalone) |
| [Issues](issues.md) | `/projects/:id/issues` (also available for groups and standalone) |
| [Iterations](iterations.md) **(PREMIUM ALL)** | `/projects/:id/iterations` (also available for groups) |
| [Iterations](iterations.md) | `/projects/:id/iterations` (also available for groups) |
| [Project CI/CD job token scope](project_job_token_scopes.md) | `/projects/:id/job_token_scope` |
| [Jobs](jobs.md) | `/projects/:id/jobs`, `/projects/:id/pipelines/.../jobs` |
| [Jobs Artifacts](job_artifacts.md) | `/projects/:id/jobs/:job_id/artifacts` |
| [Labels](labels.md) | `/projects/:id/labels` |
| [Maven repository](packages/maven.md) | `/projects/:id/packages/maven` (also available for groups and standalone) |
| [Members](members.md) | `/projects/:id/members` (also available for groups) |
| [Merge request approvals](merge_request_approvals.md) **(PREMIUM ALL)** | `/projects/:id/approvals`, `/projects/:id/merge_requests/.../approvals` |
| [Merge request approvals](merge_request_approvals.md) | `/projects/:id/approvals`, `/projects/:id/merge_requests/.../approvals` |
| [Merge requests](merge_requests.md) | `/projects/:id/merge_requests` (also available for groups and standalone) |
| [Merge trains](merge_trains.md) | `/projects/:id/merge_trains` |
| [Metadata](metadata.md) | `/metadata` |
@ -82,7 +82,7 @@ The following API resources are available in the project context:
| [Project milestones](milestones.md) | `/projects/:id/milestones` |
| [Project snippets](project_snippets.md) | `/projects/:id/snippets` |
| [Project templates](project_templates.md) | `/projects/:id/templates` |
| [Project vulnerabilities](project_vulnerabilities.md) **(ULTIMATE ALL)** | `/projects/:id/vulnerabilities` |
| [Project vulnerabilities](project_vulnerabilities.md). | `/projects/:id/vulnerabilities` |
| [Project wikis](wikis.md) | `/projects/:id/wikis` |
| [Project-level variables](project_level_variables.md) | `/projects/:id/variables` |
| [Projects](projects.md) including setting Webhooks | `/projects`, `/projects/:id/hooks` (also available for users) |
@ -103,10 +103,10 @@ The following API resources are available in the project context:
| [Tags](tags.md) | `/projects/:id/repository/tags` |
| [Terraform modules](packages/terraform-modules.md) | `/projects/:id/packages/terraform/modules` (also available standalone) |
| [User-starred metrics dashboards](metrics_user_starred_dashboards.md ) | `/projects/:id/metrics/user_starred_dashboards` |
| [Visual Review discussions](visual_review_discussions.md) **(PREMIUM ALL)** | `/projects/:id/merge_requests/:merge_request_id/visual_review_discussions` |
| [Vulnerabilities](vulnerabilities.md) **(ULTIMATE ALL)** | `/vulnerabilities/:id` |
| [Vulnerability exports](vulnerability_exports.md) **(ULTIMATE ALL)** | `/projects/:id/vulnerability_exports` |
| [Vulnerability findings](vulnerability_findings.md) **(ULTIMATE ALL)** | `/projects/:id/vulnerability_findings` |
| [Visual Review discussions](visual_review_discussions.md) | `/projects/:id/merge_requests/:merge_request_id/visual_review_discussions` |
| [Vulnerabilities](vulnerabilities.md) | `/vulnerabilities/:id` |
| [Vulnerability exports](vulnerability_exports.md) | `/projects/:id/vulnerability_exports` |
| [Vulnerability findings](vulnerability_findings.md) | `/projects/:id/vulnerability_findings` |
## Group resources
@ -120,19 +120,19 @@ The following API resources are available in the group context:
| [Debian distributions](packages/debian_group_distributions.md) | `/groups/:id/-/packages/debian` (also available for projects) |
| [Deploy tokens](deploy_tokens.md) | `/groups/:id/deploy_tokens` (also available for projects and standalone) |
| [Discussions](discussions.md) (comments and threads) | `/groups/:id/epics/.../discussions` (also available for projects) |
| [Epic issues](epic_issues.md) **(PREMIUM ALL)** | `/groups/:id/epics/.../issues` |
| [Epic links](epic_links.md) **(PREMIUM ALL)** | `/groups/:id/epics/.../epics` |
| [Epics](epics.md) **(PREMIUM ALL)** | `/groups/:id/epics` |
| [Epic issues](epic_issues.md) | `/groups/:id/epics/.../issues` |
| [Epic links](epic_links.md) | `/groups/:id/epics/.../epics` |
| [Epics](epics.md) | `/groups/:id/epics` |
| [Groups](groups.md) | `/groups`, `/groups/.../subgroups` |
| [Group badges](group_badges.md) | `/groups/:id/badges` |
| [Group issue boards](group_boards.md) | `/groups/:id/boards` |
| [Group iterations](group_iterations.md) **(PREMIUM ALL)** | `/groups/:id/iterations` (also available for projects) |
| [Group iterations](group_iterations.md) | `/groups/:id/iterations` (also available for projects) |
| [Group labels](group_labels.md) | `/groups/:id/labels` |
| [Group-level variables](group_level_variables.md) | `/groups/:id/variables` |
| [Group milestones](group_milestones.md) | `/groups/:id/milestones` |
| [Group releases](group_releases.md) | `/groups/:id/releases` |
| [Group SSH certificates](group_ssh_certificates.md) **(PREMIUM ALL)** | `/groups/:id/ssh_certificates` |
| [Group wikis](group_wikis.md) **(PREMIUM ALL)** | `/groups/:id/wikis` |
| [Group SSH certificates](group_ssh_certificates.md) | `/groups/:id/ssh_certificates` |
| [Group wikis](group_wikis.md) | `/groups/:id/wikis` |
| [Invitations](invitations.md) | `/groups/:id/invitations` (also available for projects) |
| [Issues](issues.md) | `/groups/:id/issues` (also available for projects and standalone) |
| [Issues Statistics](issues_statistics.md) | `/groups/:id/issues_statistics` (also available for projects and standalone) |
@ -151,31 +151,31 @@ The following API resources are available outside of project and group contexts
| Resource | Available endpoints |
|:---------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------|
| [Appearance](appearance.md) **(FREE SELF)** | `/application/appearance` |
| [Appearance](appearance.md) | `/application/appearance` |
| [Applications](applications.md) | `/applications` |
| [Audit Events](audit_events.md) **(PREMIUM SELF)** | `/audit_events` |
| [Audit Events](audit_events.md) | `/audit_events` |
| [Avatar](avatar.md) | `/avatar` |
| [Broadcast messages](broadcast_messages.md) | `/broadcast_messages` |
| [Code snippets](snippets.md) | `/snippets` |
| [Code Suggestions](code_suggestions.md) | `/code_suggestions` |
| [Custom attributes](custom_attributes.md) | `/users/:id/custom_attributes` (also available for groups and projects) |
| [Dependency list exports](dependency_list_export.md) **(ULTIMATE ALL)** | `/pipelines/:id/dependency_list_exports`, `/projects/:id/dependency_list_exports`, `/groups/:id/dependency_list_exports`, `/security/dependency_list_exports/:id`, `/security/dependency_list_exports/:id/download` |
| [Dependency list exports](dependency_list_export.md) | `/pipelines/:id/dependency_list_exports`, `/projects/:id/dependency_list_exports`, `/groups/:id/dependency_list_exports`, `/security/dependency_list_exports/:id`, `/security/dependency_list_exports/:id/download` |
| [Deploy keys](deploy_keys.md) | `/deploy_keys` (also available for projects) |
| [Deploy tokens](deploy_tokens.md) | `/deploy_tokens` (also available for projects and groups) |
| [Events](events.md) | `/events`, `/users/:id/events` (also available for projects) |
| [Feature flags](features.md) | `/features` |
| [Geo Nodes](geo_nodes.md) **(PREMIUM SELF)** | `/geo_nodes` |
| [Geo Nodes](geo_nodes.md) | `/geo_nodes` |
| [Group Activity Analytics](group_activity_analytics.md) | `/analytics/group_activity/{issues_count}` |
| [Group repository storage moves](group_repository_storage_moves.md) **(PREMIUM SELF)** | `/group_repository_storage_moves` |
| [Group repository storage moves](group_repository_storage_moves.md) | `/group_repository_storage_moves` |
| [Import repository from GitHub](import.md#import-repository-from-github) | `/import/github` |
| [Import repository from Bitbucket Server](import.md#import-repository-from-bitbucket-server) | `/import/bitbucket_server` |
| [Instance clusters](instance_clusters.md) **(FREE SELF)** | `/admin/clusters` |
| [Instance-level CI/CD variables](instance_level_ci_variables.md) **(FREE SELF)** | `/admin/ci/variables` |
| [Instance clusters](instance_clusters.md) | `/admin/clusters` |
| [Instance-level CI/CD variables](instance_level_ci_variables.md) | `/admin/ci/variables` |
| [Issues Statistics](issues_statistics.md) | `/issues_statistics` (also available for groups and projects) |
| [Issues](issues.md) | `/issues` (also available for groups and projects) |
| [Jobs](jobs.md) | `/job` |
| [Keys](keys.md) | `/keys` |
| [License](license.md) **(FREE SELF)** | `/license` |
| [License](license.md) | `/license` |
| [Markdown](markdown.md) | `/markdown` |
| [Merge requests](merge_requests.md) | `/merge_requests` (also available for groups and projects) |
| [Metrics dashboard annotations](metrics_dashboard_annotations.md) | `/environments/:id/metrics_dashboard/annotations`, `/clusters/:id/metrics_dashboard/annotations` |
@ -184,15 +184,15 @@ The following API resources are available outside of project and group contexts
| [Pages domains](pages_domains.md) | `/pages/domains` (also available for projects) |
| [Personal access tokens](personal_access_tokens.md) | `/personal_access_tokens` |
| [Plan limits](plan_limits.md) | `/application/plan_limits` |
| [Project repository storage moves](project_repository_storage_moves.md) **(FREE SELF)** | `/project_repository_storage_moves` |
| [Project repository storage moves](project_repository_storage_moves.md) | `/project_repository_storage_moves` |
| [Projects](projects.md) | `/users/:id/projects` (also available for projects) |
| [Runners](runners.md) | `/runners` (also available for projects) |
| [Search](search.md) | `/search` (also available for groups and projects) |
| [Service Data](usage_data.md) | `/usage_data` (For GitLab instance [Administrator](../user/permissions.md) users only) |
| [Settings](settings.md) **(FREE SELF)** | `/application/settings` |
| [Sidekiq metrics](sidekiq_metrics.md) **(FREE SELF)** | `/sidekiq` |
| [Sidekiq queues administration](admin_sidekiq_queues.md) **(FREE SELF)** | `/admin/sidekiq/queues/:queue_name` |
| [Snippet repository storage moves](snippet_repository_storage_moves.md) **(FREE SELF)** | `/snippet_repository_storage_moves` |
| [Settings](settings.md) | `/application/settings` |
| [Sidekiq metrics](sidekiq_metrics.md) | `/sidekiq` |
| [Sidekiq queues administration](admin_sidekiq_queues.md) | `/admin/sidekiq/queues/:queue_name` |
| [Snippet repository storage moves](snippet_repository_storage_moves.md) | `/snippet_repository_storage_moves` |
| [Statistics](statistics.md) | `/application/statistics` |
| [Suggestions](suggestions.md) | `/suggestions` |
| [System hooks](system_hooks.md) | `/hooks` |

1
doc/api/merge_requests.md
View File

@ -827,6 +827,7 @@ Use `detailed_merge_status` instead of `merge_status` to account for all potenti
- `not_approved`: Approval is required before merge.
- `not_open`: The merge request must be open before merge.
- `policies_denied`: The merge request contains denied policies.
- `jira_association_missing`: The title or description must reference a Jira issue.
### Preparation steps

23
doc/integration/saml.md
View File

@ -716,12 +716,23 @@ your provider's support.
### Configure assertions
| Field | Supported default keys |
|-----------------|------------------------|
| Email (required)| `email`, `mail` |
| Full Name | `name` |
| First Name | `first_name`, `firstname`, `firstName` |
| Last Name | `last_name`, `lastname`, `lastName` |
> - Microsoft Azure/Entra ID attribute support [introduced on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/420766) in GitLab 16.7.
> - Microsoft Azure/Entra ID attribute support [introduced on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/136734) in GitLab 16.7 [with a flag](../administration/feature_flags.md) named `saml_microsoft_attribute_names`. Disabled by default, and available to GitLab.com administrators only.
FLAG:
On self-managed GitLab, Microsoft Azure/Entra ID attributes are supported by default.
In the following table, these attributes begin with either `http://schemas.xmlsoap.org`
or `http://schemas.microsoft.com`.
On GitLab.com, Microsoft Azure/Entra ID attributes are introduced
[with a flag](../administration/feature_flags.md) named `saml_microsoft_attribute_names`.
On GitLab.com, this feature is unavailable but can be configured by GitLab.com administrators only.
| Field | Supported default keys |
|-----------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Email (required)| `email`, `mail`, `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress`, `http://schemas.microsoft.com/ws/2008/06/identity/claims/emailaddress` |
| Full Name | `name`, `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name`, `http://schemas.microsoft.com/ws/2008/06/identity/claims/name` |
| First Name | `first_name`, `firstname`, `firstName`, `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname`, `http://schemas.microsoft.com/ws/2008/06/identity/claims/givenname` |
| Last Name | `last_name`, `lastname`, `lastName`, `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname`, `http://schemas.microsoft.com/ws/2008/06/identity/claims/surname` |
See [`attribute_statements`](#map-saml-response-attribute-names) for:

9
doc/update/background_migrations.md
View File

@ -96,6 +96,15 @@ To query the database directly for the status of batched background migrations:
WHERE status <> 3;
```
Alternatively, you can wrap the query with `gitlab-psql -c "<QUERY>"` to check the status of
batched background migrations:
```shell
gitlab-psql -c "SELECT job_class_name, table_name, column_name, job_arguments FROM batched_background_migrations WHERE status <> 3;"
```
If the query returns zero rows, all batched background migrations are complete.
### Enable or disable advanced features
Batched background migrations provide feature flags that enable you to customize

3
doc/user/group/saml_sso/example_saml_config.md
View File

@ -41,9 +41,6 @@ This section has screenshots for the elements of Azure Active Directory configur
![Azure AD user claims](img/AzureAD-claims.png)
NOTE:
Attribute names starting with phrases such as `http://schemas.microsoft.com/ws/2008/06/identity/claims/` are not supported.
### SCIM mapping
Provisioning:

3
doc/user/group/saml_sso/index.md
View File

@ -192,9 +192,6 @@ Optionally, you can pass user information to GitLab as attributes in the SAML as
For more information, see the [attributes available for self-managed GitLab instances](../../../integration/saml.md#configure-assertions).
NOTE:
Attribute names starting with phrases such as `http://schemas.microsoft.com/ws/2008/06/identity/claims/` are not supported. For more information on configuring required attribute names in the SAML identity provider's settings, see [example group SAML and SCIM configurations](../../../user/group/saml_sso/example_saml_config.md).
### Use metadata
To configure some identity providers, you need a GitLab metadata URL.

4
lib/api/entities/commit.rb
View File

@ -17,6 +17,10 @@ module API
expose :committer_email, documentation: { type: 'string', example: 'jack@example.com' }
expose :committed_date, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' }
expose :trailers, documentation: { type: 'object', example: '{ "Merged-By": "Jane Doe janedoe@gitlab.com" }' }
expose :extended_trailers, documentation: {
type: 'object',
example: '{ "Signed-off-by": ["John Doe <johndoe@gitlab.com>", "Jane Doe <janedoe@gitlab.com>"] }'
}
expose :web_url,
documentation: {

27
lib/api/helpers/kubernetes/agent_helpers.rb
View File

@ -43,9 +43,9 @@ module API
def increment_unique_events
events = params[:unique_counters]&.slice(
:agent_users_using_ci_tunnel,
:k8s_api_proxy_requests_unique_users_via_ci_access, :k8s_api_proxy_requests_unique_agents_via_ci_access,
:k8s_api_proxy_requests_unique_users_via_user_access, :k8s_api_proxy_requests_unique_agents_via_user_access,
:k8s_api_proxy_requests_unique_users_via_pat_access, :k8s_api_proxy_requests_unique_agents_via_pat_access,
:k8s_api_proxy_requests_unique_agents_via_ci_access,
:k8s_api_proxy_requests_unique_agents_via_user_access,
:k8s_api_proxy_requests_unique_agents_via_pat_access,
:flux_git_push_notified_unique_projects
)
@ -54,6 +54,27 @@ module API
end
end
def track_unique_user_events
events = params[:unique_counters]&.slice(
:k8s_api_proxy_requests_unique_users_via_ci_access,
:k8s_api_proxy_requests_unique_users_via_user_access,
:k8s_api_proxy_requests_unique_users_via_pat_access
)
return if events.blank?
unique_user_ids = events.values.flatten.uniq
users = User.id_in(unique_user_ids).index_by(&:id)
events.each do |event, user_ids|
user_ids.each do |user_id|
user = users[user_id]
next if user.nil?
Gitlab::InternalEvents.track_event(event, user: user)
end
end
end
def increment_count_events
events = params[:counters]&.slice(
:gitops_sync, :k8s_api_proxy_request, :flux_git_push_notifications_total,

1
lib/api/internal/kubernetes.rb
View File

@ -140,6 +140,7 @@ module API
post '/', feature_category: :deployment_management do
increment_count_events
increment_unique_events
track_unique_user_events
no_content!
rescue ArgumentError => e

29
lib/gitlab/auth/saml/config.rb
View File

@ -4,10 +4,39 @@ module Gitlab
module Auth
module Saml
class Config
DEFAULT_NICKNAME_ATTRS = %w[username nickname].freeze
DEFAULT_NAME_ATTRS = %w[
http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name
http://schemas.microsoft.com/ws/2008/06/identity/claims/name
].freeze
DEFAULT_EMAIL_ATTRS = %w[
http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress
http://schemas.microsoft.com/ws/2008/06/identity/claims/emailaddress
].freeze
DEFAULT_FIRST_NAME_ATTRS = %w[
http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname
http://schemas.microsoft.com/ws/2008/06/identity/claims/givenname
].freeze
DEFAULT_LAST_NAME_ATTRS = %w[
http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname
http://schemas.microsoft.com/ws/2008/06/identity/claims/surname
].freeze
class << self
def enabled?
::AuthHelper.saml_providers.any?
end
def default_attribute_statements
defaults = OmniAuth::Strategies::SAML.default_options[:attribute_statements].to_hash.deep_symbolize_keys
defaults[:nickname] = DEFAULT_NICKNAME_ATTRS.dup
defaults[:name].concat(DEFAULT_NAME_ATTRS)
defaults[:email].concat(DEFAULT_EMAIL_ATTRS)
defaults[:first_name].concat(DEFAULT_FIRST_NAME_ATTRS)
defaults[:last_name].concat(DEFAULT_LAST_NAME_ATTRS)
defaults
end
end
DEFAULT_PROVIDER_NAME = 'saml'

11
lib/gitlab/git/commit.rb
View File

@ -28,7 +28,8 @@ module Gitlab
SERIALIZE_KEYS = [
:id, :message, :parent_ids,
:authored_date, :author_name, :author_email,
:committed_date, :committer_name, :committer_email, :trailers, :referenced_by
:committed_date, :committer_name, :committer_email,
:trailers, :extended_trailers, :referenced_by
].freeze
attr_accessor(*SERIALIZE_KEYS)
@ -432,9 +433,17 @@ module Gitlab
@committer_email = commit.committer.email.dup
@parent_ids = Array(commit.parent_ids)
@trailers = commit.trailers.to_h { |t| [t.key, t.value] }
@extended_trailers = parse_commit_trailers(commit.trailers)
@referenced_by = Array(commit.referenced_by)
end
# Turn the commit trailers into a hash of key: [value, value] arrays
def parse_commit_trailers(trailers)
trailers.each_with_object({}) do |trailer, hash|
(hash[trailer.key] ||= []) << trailer.value
end
end
# Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone
# offset in author.timezone. If the latter isn't present, assume UTC.
def init_date_from_gitaly(author)

4
lib/gitlab/github_import/job_delay_calculator.rb
View File

@ -7,7 +7,9 @@ module Gitlab
module JobDelayCalculator
# Default batch settings for parallel import (can be redefined in Importer/Worker classes)
def parallel_import_batch
{ size: 1000, delay: 1.minute }
batch_size = Feature.enabled?(:github_import_increased_concurrent_workers, project.creator) ? 5000 : 1000
{ size: batch_size, delay: 1.minute }
end
private

13
lib/gitlab/omniauth_initializer.rb
View File

@ -29,6 +29,8 @@ module Gitlab
{
authorize_params: { gl_auth_type: 'login' }
}
when ->(provider_name) { AuthHelper.saml_providers.include?(provider_name.to_sym) }
{ attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements }
else
{}
end
@ -61,7 +63,7 @@ module Gitlab
provider_arguments.concat arguments
provider_arguments << defaults unless defaults.empty?
when Hash, GitlabSettings::Options
hash_arguments = arguments.deep_symbolize_keys.deep_merge(defaults)
hash_arguments = merge_hash_defaults_and_args(defaults, arguments)
normalized = normalize_hash_arguments(hash_arguments)
# A Hash from the configuration will be passed as is.
@ -80,6 +82,15 @@ module Gitlab
provider_arguments
end
def merge_hash_defaults_and_args(defaults, arguments)
return arguments.to_hash if defaults.empty?
revert_merging = Gitlab::Utils.to_boolean(ENV['REVERT_OMNIAUTH_DEFAULT_MERGING'])
return arguments.to_hash.deep_symbolize_keys.deep_merge(defaults) if revert_merging
defaults.deep_merge(arguments.deep_symbolize_keys)
end
def normalize_hash_arguments(args)
args.deep_symbolize_keys!

2
lib/gitlab/usage_data_counters/hll_redis_counter.rb
View File

@ -66,7 +66,7 @@ module Gitlab
rescue StandardError => e
# Ignore any exceptions unless is dev or test env
# The application flow should not be blocked by erros in tracking
# The application flow should not be blocked by errors in tracking
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
end

2
qa/Gemfile
View File

@ -41,7 +41,7 @@ gem 'chemlab-library-www-gitlab-com', '~> 0.1', '>= 0.1.1'
# dependencies for jenkins client
gem 'nokogiri', '~> 1.15', '>= 1.15.5'
gem 'deprecation_toolkit', '~> 2.0.3', require: false
gem 'deprecation_toolkit', '~> 2.0.4', require: false
gem 'factory_bot', '~> 6.3.0'

4
qa/Gemfile.lock
View File

@ -75,7 +75,7 @@ GEM
crass (1.0.6)
debug_inspector (1.1.0)
declarative (0.0.20)
deprecation_toolkit (2.0.3)
deprecation_toolkit (2.0.4)
activesupport (>= 5.2)
diff-lcs (1.3)
domain_name (0.5.20190701)
@ -357,7 +357,7 @@ DEPENDENCIES
chemlab (~> 0.11, >= 0.11.1)
chemlab-library-www-gitlab-com (~> 0.1, >= 0.1.1)
confiner (~> 0.4)
deprecation_toolkit (~> 2.0.3)
deprecation_toolkit (~> 2.0.4)
factory_bot (~> 6.3.0)
faker (~> 3.2, >= 3.2.2)
faraday-retry (~> 2.2)

4
qa/qa/specs/features/api/1_manage/migration/gitlab_migration_large_project_spec.rb
View File

@ -66,7 +66,7 @@ module QA
let(:source_project) { source_group.projects(auto_paginate: true).find { |project| project.name == gitlab_source_project }.reload! }
let(:source_branches) { source_project.repository_branches(auto_paginate: true).map { |b| b[:name] } }
let(:source_commits) { source_project.commits(auto_paginate: true).map { |c| c[:id] } }
let(:source_labels) { source_project.labels(auto_paginate: true).map { |l| l.except(:id) } }
let(:source_labels) { source_project.labels(auto_paginate: true).map { |l| l.except(:id, :description_html) } }
let(:source_milestones) { source_project.milestones(auto_paginate: true).map { |ms| ms.except(:id, :web_url, :project_id) } }
let(:source_mrs) { fetch_mrs(source_project, source_api_client, transform_urls: true) }
let(:source_issues) { fetch_issues(source_project, source_api_client, transform_urls: true) }
@ -88,7 +88,7 @@ module QA
let(:imported_project) { imported_group.projects(auto_paginate: true).find { |project| project.name == gitlab_source_project }.reload! }
let(:branches) { imported_project.repository_branches(auto_paginate: true).map { |b| b[:name] } }
let(:commits) { imported_project.commits(auto_paginate: true).map { |c| c[:id] } }
let(:labels) { imported_project.labels(auto_paginate: true).map { |l| l.except(:id) } }
let(:labels) { imported_project.labels(auto_paginate: true).map { |l| l.except(:id, :description_html) } }
let(:milestones) { imported_project.milestones(auto_paginate: true).map { |ms| ms.except(:id, :web_url, :project_id) } }
let(:mrs) { fetch_mrs(imported_project, api_client) }
let(:issues) { fetch_issues(imported_project, api_client) }

10
spec/factories/gitaly/commit.rb
View File

@ -16,5 +16,15 @@ FactoryBot.define do
body { subject + "\nMy body" }
author { association(:gitaly_commit_author) }
committer { association(:gitaly_commit_author) }
trailers do
trailers = body.lines.keep_if { |l| l =~ /.*: / }.map do |l|
key, value = *l.split(":").map(&:strip)
Gitaly::CommitTrailer.new(key: key, value: value)
end
Google::Protobuf::RepeatedField.new(:message, Gitaly::CommitTrailer, trailers)
end
end
end

35
spec/lib/gitlab/auth/saml/config_spec.rb
View File

@ -19,6 +19,41 @@ RSpec.describe Gitlab::Auth::Saml::Config do
end
end
describe '.default_attribute_statements' do
it 'includes upstream defaults, nickname and Microsoft values' do
expect(described_class.default_attribute_statements).to eq(
{
nickname: %w[username nickname],
name: [
'name',
'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name',
'http://schemas.microsoft.com/ws/2008/06/identity/claims/name'
],
email: [
'email',
'mail',
'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress',
'http://schemas.microsoft.com/ws/2008/06/identity/claims/emailaddress'
],
first_name: [
'first_name',
'firstname',
'firstName',
'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname',
'http://schemas.microsoft.com/ws/2008/06/identity/claims/givenname'
],
last_name: [
'last_name',
'lastname',
'lastName',
'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname',
'http://schemas.microsoft.com/ws/2008/06/identity/claims/surname'
]
}
)
end
end
describe '#external_groups' do
let(:config_1) { described_class.new('saml1') }

25
spec/lib/gitlab/git/commit_spec.rb
View File

@ -8,7 +8,6 @@ RSpec.describe Gitlab::Git::Commit, feature_category: :source_code_management do
describe "Commit info from gitaly commit" do
let(:subject) { (+"My commit").force_encoding('ASCII-8BIT') }
let(:body) { subject + (+"My body").force_encoding('ASCII-8BIT') }
let(:body_size) { body.length }
let(:gitaly_commit) { build(:gitaly_commit, subject: subject, body: body, body_size: body_size, tree_id: tree_id) }
let(:id) { gitaly_commit.id }
@ -17,6 +16,17 @@ RSpec.describe Gitlab::Git::Commit, feature_category: :source_code_management do
let(:author) { gitaly_commit.author }
let(:commit) { described_class.new(repository, gitaly_commit) }
let(:body) do
body = +<<~BODY
Bleep bloop.
Cc: John Doe <johndoe@gitlab.com>
Cc: Jane Doe <janedoe@gitlab.com>
BODY
[subject, "\n", body].join.force_encoding("ASCII-8BIT")
end
it { expect(commit.short_id).to eq(id[0..10]) }
it { expect(commit.id).to eq(id) }
it { expect(commit.sha).to eq(id) }
@ -29,6 +39,18 @@ RSpec.describe Gitlab::Git::Commit, feature_category: :source_code_management do
it { expect(commit.parent_ids).to eq(gitaly_commit.parent_ids) }
it { expect(commit.tree_id).to eq(tree_id) }
it "parses the commit trailers" do
expect(commit.trailers).to eq(
{ "Cc" => "Jane Doe <janedoe@gitlab.com>" }
)
end
it "parses the extended commit trailers" do
expect(commit.extended_trailers).to eq(
{ "Cc" => ["John Doe <johndoe@gitlab.com>", "Jane Doe <janedoe@gitlab.com>"] }
)
end
context 'non-UTC dates' do
let(:seconds) { Time.now.to_i }
@ -773,6 +795,7 @@ RSpec.describe Gitlab::Git::Commit, feature_category: :source_code_management do
message: "tree css fixes",
parent_ids: ["874797c3a73b60d2187ed6e2fcabd289ff75171e"],
trailers: {},
extended_trailers: {},
referenced_by: []
}
end

2
spec/lib/gitlab/github_import/importer/collaborators_importer_spec.rb
View File

@ -6,7 +6,7 @@ RSpec.describe Gitlab::GithubImport::Importer::CollaboratorsImporter, feature_ca
subject(:importer) { described_class.new(project, client, parallel: parallel) }
let(:parallel) { true }
let(:project) { instance_double(Project, id: 4, import_source: 'foo/bar', import_state: nil) }
let(:project) { build(:project, id: 4, import_source: 'foo/bar', import_state: nil) }
let(:client) { instance_double(Gitlab::GithubImport::Client) }
let(:github_collaborator) do

2
spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb
View File

@ -6,7 +6,7 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNoteImporter, :aggregate_fail
let_it_be(:project) { create(:project, :repository) }
let_it_be(:user) { create(:user) }
let(:client) { double(:client) }
let(:client) { instance_double(Gitlab::GithubImport::Client) }
let(:discussion_id) { 'b0fa404393eeebb4e82becb8104f238812bb1fe6' }
let(:created_at) { Time.new(2017, 1, 1, 12, 00).utc }
let(:updated_at) { Time.new(2017, 1, 1, 12, 15).utc }

4
spec/lib/gitlab/github_import/importer/diff_notes_importer_spec.rb
View File

@ -3,8 +3,8 @@
require 'spec_helper'
RSpec.describe Gitlab::GithubImport::Importer::DiffNotesImporter, feature_category: :importers do
let(:project) { double(:project, id: 4, import_source: 'foo/bar') }
let(:client) { double(:client) }
let(:project) { build(:project, id: 4, import_source: 'foo/bar') }
let(:client) { instance_double(Gitlab::GithubImport::Client) }
let(:github_comment) do
{

2
spec/lib/gitlab/github_import/importer/issue_events_importer_spec.rb
View File

@ -5,7 +5,7 @@ require 'spec_helper'
RSpec.describe Gitlab::GithubImport::Importer::IssueEventsImporter, feature_category: :importers do
subject(:importer) { described_class.new(project, client, parallel: parallel) }
let(:project) { instance_double(Project, id: 4, import_source: 'foo/bar') }
let(:project) { build(:project, id: 4, import_source: 'foo/bar') }
let(:client) { instance_double(Gitlab::GithubImport::Client) }
let(:parallel) { true }

4
spec/lib/gitlab/github_import/importer/issues_importer_spec.rb
View File

@ -3,8 +3,8 @@
require 'spec_helper'
RSpec.describe Gitlab::GithubImport::Importer::IssuesImporter, feature_category: :importers do
let(:project) { double(:project, id: 4, import_source: 'foo/bar') }
let(:client) { double(:client) }
let(:project) { build(:project, id: 4, import_source: 'foo/bar') }
let(:client) { instance_double(Gitlab::GithubImport::Client) }
let(:created_at) { Time.new(2017, 1, 1, 12, 00) }
let(:updated_at) { Time.new(2017, 1, 1, 12, 15) }

4
spec/lib/gitlab/github_import/importer/notes_importer_spec.rb
View File

@ -3,8 +3,8 @@
require 'spec_helper'
RSpec.describe Gitlab::GithubImport::Importer::NotesImporter, feature_category: :importers do
let(:project) { double(:project, id: 4, import_source: 'foo/bar') }
let(:client) { double(:client) }
let(:project) { build(:project, id: 4, import_source: 'foo/bar') }
let(:client) { instance_double(Gitlab::GithubImport::Client) }
let(:github_comment) do
{

6
spec/lib/gitlab/github_import/importer/protected_branches_importer_spec.rb
View File

@ -5,8 +5,8 @@ require 'spec_helper'
RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchesImporter, feature_category: :importers do
subject(:importer) { described_class.new(project, client, parallel: parallel) }
let(:project) { instance_double('Project', id: 4, import_source: 'foo/bar') }
let(:client) { instance_double('Gitlab::GithubImport::Client') }
let(:project) { build(:project, id: 4, import_source: 'foo/bar') }
let(:client) { instance_double(Gitlab::GithubImport::Client) }
let(:parallel) { true }
let(:branches) do
@ -112,7 +112,7 @@ RSpec.describe Gitlab::GithubImport::Importer::ProtectedBranchesImporter, featur
end
it 'imports each protected branch in sequence' do
protected_branch_importer = instance_double('Gitlab::GithubImport::Importer::ProtectedBranchImporter')
protected_branch_importer = instance_double(Gitlab::GithubImport::Importer::ProtectedBranchImporter)
expect(Gitlab::GithubImport::Importer::ProtectedBranchImporter)
.to receive(:new)

33
spec/lib/gitlab/github_import/job_delay_calculator_spec.rb Normal file
View File

@ -0,0 +1,33 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::GithubImport::JobDelayCalculator, feature_category: :importers do
let(:project) { build(:project) }
let(:importer_class) do
Class.new do
attr_reader :project
def initialize(project)
@project = project
end
include Gitlab::GithubImport::JobDelayCalculator
end
end
describe "#parallel_import_batch" do
subject { importer_class.new(project).parallel_import_batch }
it { is_expected.to eq({ size: 5000, delay: 1.minute }) }
context 'when `github_import_increased_concurrent_workers` feature flag is disabled' do
before do
stub_feature_flags(github_import_increased_concurrent_workers: false)
end
it { is_expected.to eq({ size: 1000, delay: 1.minute }) }
end
end
end

117
spec/lib/gitlab/omniauth_initializer_spec.rb
View File

@ -2,7 +2,9 @@
require 'spec_helper'
RSpec.describe Gitlab::OmniauthInitializer do
RSpec.describe Gitlab::OmniauthInitializer, feature_category: :system_access do
include LoginHelpers
let(:devise_config) { class_double(Devise) }
subject(:initializer) { described_class.new(devise_config) }
@ -224,6 +226,119 @@ RSpec.describe Gitlab::OmniauthInitializer do
subject.execute([shibboleth_config])
end
context 'when SAML providers are configured' do
it 'configures default args for a single SAML provider' do
stub_omniauth_config(providers: [{ name: 'saml', args: { idp_sso_service_url: 'https://saml.example.com' } }])
expect(devise_config).to receive(:omniauth).with(
:saml,
{
idp_sso_service_url: 'https://saml.example.com',
attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements
}
)
initializer.execute(Gitlab.config.omniauth.providers)
end
context 'when configuration provides matching keys' do
before do
stub_omniauth_config(
providers: [
{
name: 'saml',
args: { idp_sso_service_url: 'https://saml.example.com', attribute_statements: { email: ['custom_attr'] } }
}
]
)
end
it 'merges arguments with user configuration preference' do
expect(devise_config).to receive(:omniauth).with(
:saml,
{
idp_sso_service_url: 'https://saml.example.com',
attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements
.merge({ email: ['custom_attr'] })
}
)
initializer.execute(Gitlab.config.omniauth.providers)
end
it 'merges arguments with defaults preference when REVERT_OMNIAUTH_DEFAULT_MERGING is true' do
stub_env('REVERT_OMNIAUTH_DEFAULT_MERGING', 'true')
expect(devise_config).to receive(:omniauth).with(
:saml,
{
idp_sso_service_url: 'https://saml.example.com',
attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements
}
)
initializer.execute(Gitlab.config.omniauth.providers)
end
end
it 'configures defaults args for multiple SAML providers' do
stub_omniauth_config(
providers: [
{ name: 'saml', args: { idp_sso_service_url: 'https://saml.example.com' } },
{
name: 'saml2',
args: { strategy_class: 'OmniAuth::Strategies::SAML', idp_sso_service_url: 'https://saml2.example.com' }
}
]
)
expect(devise_config).to receive(:omniauth).with(
:saml,
{
idp_sso_service_url: 'https://saml.example.com',
attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements
}
)
expect(devise_config).to receive(:omniauth).with(
:saml2,
{
idp_sso_service_url: 'https://saml2.example.com',
strategy_class: OmniAuth::Strategies::SAML,
attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements
}
)
initializer.execute(Gitlab.config.omniauth.providers)
end
it 'merges arguments with user configuration preference for custom SAML provider' do
stub_omniauth_config(
providers: [
{
name: 'custom_saml',
args: {
strategy_class: 'OmniAuth::Strategies::SAML',
idp_sso_service_url: 'https://saml2.example.com',
attribute_statements: { email: ['custom_attr'] }
}
}
]
)
expect(devise_config).to receive(:omniauth).with(
:custom_saml,
{
idp_sso_service_url: 'https://saml2.example.com',
strategy_class: OmniAuth::Strategies::SAML,
attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements
.merge({ email: ['custom_attr'] })
}
)
initializer.execute(Gitlab.config.omniauth.providers)
end
end
it 'configures defaults for google_oauth2' do
google_config = {
'name' => 'google_oauth2',

10
spec/models/ci/pipeline_metadata_spec.rb
View File

@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Ci::PipelineMetadata do
RSpec.describe Ci::PipelineMetadata, feature_category: :pipeline_composition do
it { is_expected.to belong_to(:project) }
it { is_expected.to belong_to(:pipeline) }
@ -10,5 +10,13 @@ RSpec.describe Ci::PipelineMetadata do
it { is_expected.to validate_length_of(:name).is_at_least(1).is_at_most(255) }
it { is_expected.to validate_presence_of(:project) }
it { is_expected.to validate_presence_of(:pipeline) }
it do
is_expected.to define_enum_for(
:auto_cancel_on_new_commit
).with_values(
conservative: 0, interruptible: 1, disabled: 2
).with_prefix
end
end
end

28
spec/requests/api/commits_spec.rb
View File

@ -44,6 +44,30 @@ RSpec.describe API::Commits, feature_category: :source_code_management do
expect(response).to include_limited_pagination_headers
end
describe "commit trailers" do
it "doesn't include the commit trailers by default" do
get api(route, current_user), params: { page: 2 }
commit_with_trailers = json_response.find { |c| c["trailers"].any? }
expect(commit_with_trailers).to be_nil
expect(json_response.first["trailers"]).to eq({})
end
it "does include the commit trailers when specified in the params" do
# Test repo commits with trailers are further down the list, so use a
# higher page number.
get api(route, current_user), params: { page: 2, trailers: true }
commit_with_trailers = json_response.find { |c| c["trailers"].any? }
expect(commit_with_trailers["trailers"]).to be_a(Hash)
expect(commit_with_trailers["extended_trailers"]).to be_a(Hash)
expect(commit_with_trailers["trailers"].size).to be > 0
expect(commit_with_trailers["extended_trailers"].size).to be > 0
end
end
end
context 'when unauthenticated', 'and project is public' do
@ -426,6 +450,10 @@ RSpec.describe API::Commits, feature_category: :source_code_management do
expect(commit['trailers']).to eq(
'Signed-off-by' => 'Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>'
)
expect(commit['extended_trailers']).to eq(
'Signed-off-by' => ['Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>']
)
end
end
end

18
spec/requests/api/internal/kubernetes_spec.rb
View File

@ -122,15 +122,17 @@ RSpec.describe API::Internal::Kubernetes, feature_category: :deployment_manageme
k8s_api_proxy_requests_via_user_access: 44,
k8s_api_proxy_requests_via_pat_access: 45
}
users = create_list(:user, 3)
user_ids = users.map(&:id) << users[0].id
unique_counters = {
agent_users_using_ci_tunnel: [10, 999, 777, 10],
k8s_api_proxy_requests_unique_users_via_ci_access: [10, 999, 777, 10],
k8s_api_proxy_requests_unique_agents_via_ci_access: [10, 999, 777, 10],
k8s_api_proxy_requests_unique_users_via_user_access: [10, 999, 777, 10],
k8s_api_proxy_requests_unique_agents_via_user_access: [10, 999, 777, 10],
k8s_api_proxy_requests_unique_users_via_pat_access: [10, 999, 777, 10],
k8s_api_proxy_requests_unique_agents_via_pat_access: [10, 999, 777, 10],
flux_git_push_notified_unique_projects: [10, 999, 777, 10]
agent_users_using_ci_tunnel: user_ids,
k8s_api_proxy_requests_unique_users_via_ci_access: user_ids,
k8s_api_proxy_requests_unique_agents_via_ci_access: user_ids,
k8s_api_proxy_requests_unique_users_via_user_access: user_ids,
k8s_api_proxy_requests_unique_agents_via_user_access: user_ids,
k8s_api_proxy_requests_unique_users_via_pat_access: user_ids,
k8s_api_proxy_requests_unique_agents_via_pat_access: user_ids,
flux_git_push_notified_unique_projects: user_ids
}
expected_counters = {
kubernetes_agent_gitops_sync: request_count * counters[:gitops_sync],