diff --git a/GITLAB_KAS_VERSION b/GITLAB_KAS_VERSION index 0c8e9fbc560..e2ed64bc552 100644 --- a/GITLAB_KAS_VERSION +++ b/GITLAB_KAS_VERSION @@ -1 +1 @@ -fe81274a218800d7e46002719c712915c8e491bf +5ae422a93e86dc713f0c35b794bfca6e9ed31cc7 diff --git a/Gemfile.checksum b/Gemfile.checksum index de8990ddf4b..09d07d845ab 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -28,7 +28,7 @@ {"name":"asciidoctor-kroki","version":"0.10.0","platform":"ruby","checksum":"8e4225d88f120e2e7b5d3f5ddb67c5e69496d7344a16c57db5036ac900123062"}, {"name":"asciidoctor-plantuml","version":"0.0.16","platform":"ruby","checksum":"407e47cd1186ded5ccc75f0c812e5524c26c571d542247c5132abb8f47bd1793"}, {"name":"ast","version":"2.4.2","platform":"ruby","checksum":"1e280232e6a33754cde542bc5ef85520b74db2aac73ec14acef453784447cc12"}, -{"name":"async","version":"2.23.0","platform":"ruby","checksum":"8323f3942046fcf206eac256954b419f0933a449d997b0fca926f9d118eb495c"}, +{"name":"async","version":"2.23.1","platform":"ruby","checksum":"612c97346948a5dbfb6b4aef12976416b01aef48ec2d41677efb25c8c32a5006"}, {"name":"atlassian-jwt","version":"0.2.1","platform":"ruby","checksum":"2fd2d87418773f2e140c038cb22e049069708aff2bd0a423a7e1740574e97823"}, {"name":"attr_required","version":"1.0.2","platform":"ruby","checksum":"f0ebfc56b35e874f4d0ae799066dbc1f81efefe2364ca3803dc9ea6a4de6cb99"}, {"name":"awesome_print","version":"1.9.2","platform":"ruby","checksum":"e99b32b704acff16d768b3468680793ced40bfdc4537eb07e06a4be11133786e"}, diff --git a/Gemfile.lock b/Gemfile.lock index 81daa188835..fa955a64637 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -344,7 +344,7 @@ GEM asciidoctor-plantuml (0.0.16) asciidoctor (>= 2.0.17, < 3.0.0) ast (2.4.2) - async (2.23.0) + async (2.23.1) console (~> 1.29) fiber-annotation io-event (~> 1.9) diff --git a/Gemfile.next.checksum b/Gemfile.next.checksum index e4edb482ec8..04488fdb7be 100644 --- a/Gemfile.next.checksum +++ b/Gemfile.next.checksum @@ -28,7 +28,7 @@ {"name":"asciidoctor-kroki","version":"0.10.0","platform":"ruby","checksum":"8e4225d88f120e2e7b5d3f5ddb67c5e69496d7344a16c57db5036ac900123062"}, {"name":"asciidoctor-plantuml","version":"0.0.16","platform":"ruby","checksum":"407e47cd1186ded5ccc75f0c812e5524c26c571d542247c5132abb8f47bd1793"}, {"name":"ast","version":"2.4.2","platform":"ruby","checksum":"1e280232e6a33754cde542bc5ef85520b74db2aac73ec14acef453784447cc12"}, -{"name":"async","version":"2.23.0","platform":"ruby","checksum":"8323f3942046fcf206eac256954b419f0933a449d997b0fca926f9d118eb495c"}, +{"name":"async","version":"2.23.1","platform":"ruby","checksum":"612c97346948a5dbfb6b4aef12976416b01aef48ec2d41677efb25c8c32a5006"}, {"name":"atlassian-jwt","version":"0.2.1","platform":"ruby","checksum":"2fd2d87418773f2e140c038cb22e049069708aff2bd0a423a7e1740574e97823"}, {"name":"attr_required","version":"1.0.2","platform":"ruby","checksum":"f0ebfc56b35e874f4d0ae799066dbc1f81efefe2364ca3803dc9ea6a4de6cb99"}, {"name":"awesome_print","version":"1.9.2","platform":"ruby","checksum":"e99b32b704acff16d768b3468680793ced40bfdc4537eb07e06a4be11133786e"}, diff --git a/Gemfile.next.lock b/Gemfile.next.lock index 0157abab51c..f1aa364ef3f 100644 --- a/Gemfile.next.lock +++ b/Gemfile.next.lock @@ -356,7 +356,7 @@ GEM asciidoctor-plantuml (0.0.16) asciidoctor (>= 2.0.17, < 3.0.0) ast (2.4.2) - async (2.23.0) + async (2.23.1) console (~> 1.29) fiber-annotation io-event (~> 1.9) diff --git a/app/models/issue.rb b/app/models/issue.rb index 7fc830afd4b..8907ad9f773 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -211,19 +211,18 @@ class Issue < ApplicationRecord ) } scope :with_issue_type, ->(types) { - types = Array(types) + type_ids = Array(types).filter_map do |type| + WorkItems::Type::BASE_TYPES.dig(type.to_sym, :id) + end - # Using != 1 since we also want the guard clause to handle empty arrays - return joins(:work_item_type).where(work_item_types: { base_type: types }) if types.size != 1 - - # This optimization helps the planer use the correct indexes when filtering by a single type - where( - '"issues"."work_item_type_id" = (?)', - WorkItems::Type.by_type(types.first).select(:id).limit(1) - ) + where(work_item_type_id: type_ids) } scope :without_issue_type, ->(types) { - joins(:work_item_type).where.not(work_item_types: { base_type: types }) + type_ids = Array(types).filter_map do |type| + WorkItems::Type::BASE_TYPES.dig(type.to_sym, :id) + end + + where.not(work_item_type_id: type_ids) } scope :public_only, -> { where(confidential: false) } diff --git a/app/services/group_access_tokens/rotate_service.rb b/app/services/group_access_tokens/rotate_service.rb index 0e52d720aeb..162047193ed 100644 --- a/app/services/group_access_tokens/rotate_service.rb +++ b/app/services/group_access_tokens/rotate_service.rb @@ -16,5 +16,17 @@ module GroupAccessTokens token_access_level <= current_user_access_level end + + private + + override :track_rotation_event + def track_rotation_event + track_internal_event( + 'rotate_grat', + user: target_user, + namespace: group, + project: nil + ) + end end end diff --git a/app/services/personal_access_tokens/rotate_service.rb b/app/services/personal_access_tokens/rotate_service.rb index e8dae3a5dd5..695de11532c 100644 --- a/app/services/personal_access_tokens/rotate_service.rb +++ b/app/services/personal_access_tokens/rotate_service.rb @@ -2,6 +2,8 @@ module PersonalAccessTokens class RotateService + include Gitlab::InternalEventsTracking + EXPIRATION_PERIOD = 1.week def initialize(current_user, token, resource = nil, params = {}) @@ -26,6 +28,8 @@ module PersonalAccessTokens response = create_access_token raise ActiveRecord::Rollback unless response.success? + + track_rotation_event end response @@ -107,6 +111,15 @@ module PersonalAccessTokens def default_expiration_date EXPIRATION_PERIOD.from_now.to_date end + + def track_rotation_event + track_internal_event( + "rotate_pat", + user: target_user, + namespace: target_user.namespace, + project: nil + ) + end end end diff --git a/app/services/project_access_tokens/rotate_service.rb b/app/services/project_access_tokens/rotate_service.rb index c8a4454faf3..e0b14e154ef 100644 --- a/app/services/project_access_tokens/rotate_service.rb +++ b/app/services/project_access_tokens/rotate_service.rb @@ -16,5 +16,17 @@ module ProjectAccessTokens token_access_level <= current_user_access_level end + + private + + override :track_rotation_event + def track_rotation_event + track_internal_event( + 'rotate_prat', + user: target_user, + namespace: project.namespace, + project: project + ) + end end end diff --git a/config/events/rotate_grat.yml b/config/events/rotate_grat.yml new file mode 100644 index 00000000000..09327d60258 --- /dev/null +++ b/config/events/rotate_grat.yml @@ -0,0 +1,17 @@ +--- +description: Rotation of a group access token +internal_events: true +action: rotate_grat +identifiers: +- project +- namespace +- user +product_group: authentication +product_categories: +- system_access +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +tiers: +- free +- premium +- ultimate diff --git a/config/events/rotate_pat.yml b/config/events/rotate_pat.yml new file mode 100644 index 00000000000..3141cac26f7 --- /dev/null +++ b/config/events/rotate_pat.yml @@ -0,0 +1,17 @@ +--- +description: Rotation of a personal access token +internal_events: true +action: rotate_pat +identifiers: +- project +- namespace +- user +product_group: authentication +product_categories: +- system_access +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +tiers: +- free +- premium +- ultimate diff --git a/config/events/rotate_prat.yml b/config/events/rotate_prat.yml new file mode 100644 index 00000000000..40d0bae46fc --- /dev/null +++ b/config/events/rotate_prat.yml @@ -0,0 +1,17 @@ +--- +description: Rotation of a project access token +internal_events: true +action: rotate_prat +identifiers: +- project +- namespace +- user +product_group: authentication +product_categories: +- system_access +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +tiers: +- free +- premium +- ultimate diff --git a/config/events/use_admin_token_api.yml b/config/events/use_admin_token_api.yml new file mode 100644 index 00000000000..242d06bcacc --- /dev/null +++ b/config/events/use_admin_token_api.yml @@ -0,0 +1,16 @@ +--- +description: Usage of the admin_token api +internal_events: true +action: use_admin_token_api +identifiers: +- namespace +- user +product_group: authentication +product_categories: +- system_access +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +tiers: +- free +- premium +- ultimate diff --git a/config/metrics/counts_all/count_distinct_namespace_id_from_rotate_grat.yml b/config/metrics/counts_all/count_distinct_namespace_id_from_rotate_grat.yml new file mode 100644 index 00000000000..261a3376f42 --- /dev/null +++ b/config/metrics/counts_all/count_distinct_namespace_id_from_rotate_grat.yml @@ -0,0 +1,23 @@ +--- +key_path: redis_hll_counters.count_distinct_namespace_id_from_rotate_grat +description: Count of unique namespaces which rotated a group access token +product_group: authentication +product_categories: +- system_access +performance_indicator_type: [] +value_type: number +status: active +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +time_frame: +- 28d +- 7d +data_source: internal_events +data_category: optional +tiers: +- free +- premium +- ultimate +events: +- name: rotate_grat + unique: namespace.id diff --git a/config/metrics/counts_all/count_distinct_namespace_id_from_rotate_pat.yml b/config/metrics/counts_all/count_distinct_namespace_id_from_rotate_pat.yml new file mode 100644 index 00000000000..71ccbb2ce56 --- /dev/null +++ b/config/metrics/counts_all/count_distinct_namespace_id_from_rotate_pat.yml @@ -0,0 +1,23 @@ +--- +key_path: redis_hll_counters.count_distinct_namespace_id_from_rotate_pat +description: Count of unique namespaces which rotated a personal access token +product_group: authentication +product_categories: +- system_access +performance_indicator_type: [] +value_type: number +status: active +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +time_frame: +- 28d +- 7d +data_source: internal_events +data_category: optional +tiers: +- free +- premium +- ultimate +events: +- name: rotate_pat + unique: namespace.id diff --git a/config/metrics/counts_all/count_distinct_namespace_id_from_rotate_prat.yml b/config/metrics/counts_all/count_distinct_namespace_id_from_rotate_prat.yml new file mode 100644 index 00000000000..004cac981cf --- /dev/null +++ b/config/metrics/counts_all/count_distinct_namespace_id_from_rotate_prat.yml @@ -0,0 +1,23 @@ +--- +key_path: redis_hll_counters.count_distinct_namespace_id_from_rotate_prat +description: Count of unique namespaces which rotated a project access token +product_group: authentication +product_categories: +- system_access +performance_indicator_type: [] +value_type: number +status: active +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +time_frame: +- 28d +- 7d +data_source: internal_events +data_category: optional +tiers: +- free +- premium +- ultimate +events: +- name: rotate_prat + unique: namespace.id diff --git a/config/metrics/counts_all/count_distinct_namespace_id_from_token_management_actions.yml b/config/metrics/counts_all/count_distinct_namespace_id_from_token_management_actions.yml new file mode 100644 index 00000000000..30223dae5ff --- /dev/null +++ b/config/metrics/counts_all/count_distinct_namespace_id_from_token_management_actions.yml @@ -0,0 +1,29 @@ +--- +key_path: redis_hll_counters.count_distinct_namespace_id_from_token_management_actions +description: Count of unique namespaces with a token management action (token rotation and admin_token_api usage) +product_group: authentication +product_categories: +- system_access +performance_indicator_type: [] +value_type: number +status: active +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +time_frame: +- 28d +- 7d +data_source: internal_events +data_category: optional +tiers: +- free +- premium +- ultimate +events: +- name: rotate_grat + unique: namespace.id +- name: rotate_pat + unique: namespace.id +- name: rotate_prat + unique: namespace.id +- name: use_admin_token_api + unique: namespace.id diff --git a/config/metrics/counts_all/count_distinct_namespace_id_from_use_admin_token_api.yml b/config/metrics/counts_all/count_distinct_namespace_id_from_use_admin_token_api.yml new file mode 100644 index 00000000000..3a6f0834783 --- /dev/null +++ b/config/metrics/counts_all/count_distinct_namespace_id_from_use_admin_token_api.yml @@ -0,0 +1,23 @@ +--- +key_path: redis_hll_counters.count_distinct_namespace_id_from_use_admin_token_api +description: Count of unique namespaces which used the admin_token api +product_group: authentication +product_categories: +- system_access +performance_indicator_type: [] +value_type: number +status: active +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +time_frame: +- 28d +- 7d +data_source: internal_events +data_category: optional +tiers: +- free +- premium +- ultimate +events: +- name: use_admin_token_api + unique: namespace.id diff --git a/config/metrics/counts_all/count_distinct_user_id_from_rotate_grat.yml b/config/metrics/counts_all/count_distinct_user_id_from_rotate_grat.yml new file mode 100644 index 00000000000..4a6c294b6d7 --- /dev/null +++ b/config/metrics/counts_all/count_distinct_user_id_from_rotate_grat.yml @@ -0,0 +1,23 @@ +--- +key_path: redis_hll_counters.count_distinct_user_id_from_rotate_grat +description: Count of unique users who rotated a group access token +product_group: authentication +product_categories: +- system_access +performance_indicator_type: [] +value_type: number +status: active +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +time_frame: +- 28d +- 7d +data_source: internal_events +data_category: optional +tiers: +- free +- premium +- ultimate +events: +- name: rotate_grat + unique: user.id diff --git a/config/metrics/counts_all/count_distinct_user_id_from_rotate_pat.yml b/config/metrics/counts_all/count_distinct_user_id_from_rotate_pat.yml new file mode 100644 index 00000000000..60332fbd718 --- /dev/null +++ b/config/metrics/counts_all/count_distinct_user_id_from_rotate_pat.yml @@ -0,0 +1,23 @@ +--- +key_path: redis_hll_counters.count_distinct_user_id_from_rotate_pat +description: Count of unique users who rotated a personal access token +product_group: authentication +product_categories: +- system_access +performance_indicator_type: [] +value_type: number +status: active +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +time_frame: +- 28d +- 7d +data_source: internal_events +data_category: optional +tiers: +- free +- premium +- ultimate +events: +- name: rotate_pat + unique: user.id diff --git a/config/metrics/counts_all/count_distinct_user_id_from_rotate_prat.yml b/config/metrics/counts_all/count_distinct_user_id_from_rotate_prat.yml new file mode 100644 index 00000000000..59b8a052bf0 --- /dev/null +++ b/config/metrics/counts_all/count_distinct_user_id_from_rotate_prat.yml @@ -0,0 +1,23 @@ +--- +key_path: redis_hll_counters.count_distinct_user_id_from_rotate_prat +description: Count of unique users who rotated a project access token +product_group: authentication +product_categories: +- system_access +performance_indicator_type: [] +value_type: number +status: active +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +time_frame: +- 28d +- 7d +data_source: internal_events +data_category: optional +tiers: +- free +- premium +- ultimate +events: +- name: rotate_prat + unique: user.id diff --git a/config/metrics/counts_all/count_distinct_user_id_from_token_management_actions.yml b/config/metrics/counts_all/count_distinct_user_id_from_token_management_actions.yml new file mode 100644 index 00000000000..d1e54eb7246 --- /dev/null +++ b/config/metrics/counts_all/count_distinct_user_id_from_token_management_actions.yml @@ -0,0 +1,29 @@ +--- +key_path: redis_hll_counters.count_distinct_user_id_from_token_management_actions +description: Count of unique users with a token management action (token rotation and admin_token_api usage) +product_group: authentication +product_categories: +- system_access +performance_indicator_type: [] +value_type: number +status: active +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +time_frame: +- 28d +- 7d +data_source: internal_events +data_category: optional +tiers: +- free +- premium +- ultimate +events: +- name: rotate_grat + unique: user.id +- name: rotate_pat + unique: user.id +- name: rotate_prat + unique: user.id +- name: use_admin_token_api + unique: user.id diff --git a/config/metrics/counts_all/count_distinct_user_id_from_use_admin_token_api.yml b/config/metrics/counts_all/count_distinct_user_id_from_use_admin_token_api.yml new file mode 100644 index 00000000000..108247b5f21 --- /dev/null +++ b/config/metrics/counts_all/count_distinct_user_id_from_use_admin_token_api.yml @@ -0,0 +1,23 @@ +--- +key_path: redis_hll_counters.count_distinct_user_id_from_use_admin_token_api +description: Count of unique users who used the admin_token api +product_group: authentication +product_categories: +- system_access +performance_indicator_type: [] +value_type: number +status: active +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +time_frame: +- 28d +- 7d +data_source: internal_events +data_category: optional +tiers: +- free +- premium +- ultimate +events: +- name: use_admin_token_api + unique: user.id diff --git a/config/metrics/counts_all/count_total_token_management_actions.yml b/config/metrics/counts_all/count_total_token_management_actions.yml new file mode 100644 index 00000000000..418127697a2 --- /dev/null +++ b/config/metrics/counts_all/count_total_token_management_actions.yml @@ -0,0 +1,26 @@ +--- +key_path: counts.count_total_token_management_actions +description: Count of token management actions (token rotation and admin_token_api usage) +product_group: authentication +product_categories: +- system_access +performance_indicator_type: [] +value_type: number +status: active +milestone: '17.10' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184008 +time_frame: +- 28d +- 7d +- all +data_source: internal_events +data_category: optional +tiers: +- free +- premium +- ultimate +events: +- name: rotate_grat +- name: rotate_pat +- name: rotate_prat +- name: use_admin_token_api diff --git a/db/docs/ci_build_pending_states.yml b/db/docs/ci_build_pending_states.yml index 9f1f491f14c..a2fe237491a 100644 --- a/db/docs/ci_build_pending_states.yml +++ b/db/docs/ci_build_pending_states.yml @@ -8,15 +8,6 @@ description: TODO introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41585 milestone: '13.4' gitlab_schema: gitlab_ci -desired_sharding_key: - project_id: - references: projects - backfill_via: - parent: - foreign_key: build_id - table: p_ci_builds - sharding_key: project_id - belongs_to: build - foreign_key_name: fk_861cd17da3_p -desired_sharding_key_migration_job_name: BackfillCiBuildPendingStatesProjectId table_size: small +sharding_key: + project_id: projects diff --git a/db/post_migrate/20250310082537_add_ci_build_pending_states_project_id_not_null.rb b/db/post_migrate/20250310082537_add_ci_build_pending_states_project_id_not_null.rb new file mode 100644 index 00000000000..8898ab7e852 --- /dev/null +++ b/db/post_migrate/20250310082537_add_ci_build_pending_states_project_id_not_null.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class AddCiBuildPendingStatesProjectIdNotNull < Gitlab::Database::Migration[2.2] + milestone '17.11' + disable_ddl_transaction! + + def up + add_not_null_constraint :ci_build_pending_states, :project_id + end + + def down + remove_not_null_constraint :ci_build_pending_states, :project_id + end +end diff --git a/db/schema_migrations/20250310082537 b/db/schema_migrations/20250310082537 new file mode 100644 index 00000000000..0990609c08e --- /dev/null +++ b/db/schema_migrations/20250310082537 @@ -0,0 +1 @@ +3f4f8e5784fffbfb9f82f71f30f600398889348f10b3ab829823b7775673ddbe \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 8da8d90f28b..f02ba06e914 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -10448,7 +10448,8 @@ CREATE TABLE ci_build_pending_states ( trace_checksum bytea, trace_bytesize bigint, partition_id bigint NOT NULL, - project_id bigint + project_id bigint, + CONSTRAINT check_20b28e5e16 CHECK ((project_id IS NOT NULL)) ); CREATE SEQUENCE ci_build_pending_states_id_seq diff --git a/doc/api/graphql/reference/_index.md b/doc/api/graphql/reference/_index.md index 6b51e63dfcc..3ee66035371 100644 --- a/doc/api/graphql/reference/_index.md +++ b/doc/api/graphql/reference/_index.md @@ -7709,6 +7709,33 @@ Input type: `MemberRoleAdminCreateInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `memberRole` | [`AdminMemberRole`](#adminmemberrole) | Member role. | +### `Mutation.memberRoleAdminDelete` + +{{< details >}} +**Introduced** in GitLab 17.10. +**Status**: Experiment. +{{< /details >}} + +Input type: `MemberRoleAdminDeleteInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `description` | [`String`](#string) | Description of the member role. | +| `id` | [`MemberRoleID!`](#memberroleid) | ID of the admin member role to delete. | +| `name` | [`String`](#string) | Name of the member role. | +| `permissions` | [`[MemberRoleAdminPermission!]`](#memberroleadminpermission) | List of all customizable admin permissions. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `memberRole` | [`AdminMemberRole`](#adminmemberrole) | Member role. | + ### `Mutation.memberRoleAdminUpdate` {{< details >}} diff --git a/doc/development/sidekiq/_index.md b/doc/development/sidekiq/_index.md index 95d7615eb06..d79aad38a28 100644 --- a/doc/development/sidekiq/_index.md +++ b/doc/development/sidekiq/_index.md @@ -413,3 +413,41 @@ tests should be placed in `spec/workers`. The application should minimise interaction with of any `Sidekiq.redis` and Sidekiq [APIs](https://github.com/mperham/sidekiq/blob/main/lib/sidekiq/api.rb). Such interactions in generic application logic should be abstracted to a [Sidekiq middleware](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/sidekiq_middleware) for re-use across teams. By decoupling application logic from Sidekiq datastore, it allows for greater freedom when horizontally scaling the GitLab background processing setup. Some exceptions to this rule would be migration-related logic or administration operations. + +## Job duration limit + +In general it is best-practice for Sidekiq jobs to run for short durations. + +Although there is no specific hard limit for job duration, there are two special considerations for long running jobs: + +1. Job durations above our [`urgency` attribute](worker_attributes.md#job-urgency) thresholds contribute negatively to + [Sidekiq Apdex](../application_slis/sidekiq_execution.md) and can impact error budgets. +1. Deploys interrupt long-running jobs. On GitLab.com, deploys can happen several times a day, which can [effectively limit the length a job can run](#effect-of-deploys-on-job-duration). + +### Effect of deploys on job duration + +During a deploy, Sidekiq is given a `TERM` signal. Jobs are given 25 seconds to finish, after which they are +interrupted and forced to stop. The 25 second grace period is the +[Sidekiq default](https://github.com/sidekiq/sidekiq/blob/ba51d286d821777fbe87ea0eff8b04f212aeadf5/lib/sidekiq/config.rb#L18) but can be +[configured through the charts](https://gitlab.com/gitlab-com/gl-infra/k8s-workloads/gitlab-com/blob/d2bb7cca2130cd9859e5d40e5bd90f5ef061d422/vendor/charts/gitlab/gprd/charts/gitlab/charts/sidekiq/values.yaml#L291). + +If a job is forced to stop a certain number of times (3 times by default, configurable +through `max_retries_after_interruption`), they are permanently killed. This happens through +our [`sidekiq-reliable-fetch` gem](https://gitlab.com/gitlab-org/gitlab/-/blob/master/vendor/gems/sidekiq-reliable-fetch/README.md). + +This effectively puts a limit on the length of time a job can run +to a span of `max_retries_after_interruption` deploys, or 3 deploys by default. + +### Tips for handling jobs with long durations + +Instead of having one big job, it's better to have many small jobs. + +To decide if a worker needs to be split up and parallelized we can look at the runtime of jobs in the logs. +If the 99th percentile of the job duration is lower than the target for that shard based on the configured +[urgency](worker_attributes.md#job-urgency), there is no need to break up the job. + +When breaking up long running jobs into many smaller jobs, do take into account downstream dependencies. +For example, if we schedule thousands of jobs that all need to write to the primary database, this +could create contention on connections to the primary database causing other Sidekiq jobs on the shard to +have to wait to obtain a connection. To circumvent this, we can consider specifying a +[concurrency limit](worker_attributes.md#concurrency-limit). diff --git a/doc/user/project/import/_index.md b/doc/user/project/import/_index.md index b40c84d8f4f..aed71a23c0e 100644 --- a/doc/user/project/import/_index.md +++ b/doc/user/project/import/_index.md @@ -130,7 +130,9 @@ GitLab.com and GitLab Self-Managed. For information on the other method available for GitLab Self-Managed with disabled feature flags, see the documentation for each importer. -User contribution mapping is not supported when you import projects to a personal namespace. +User contribution mapping is not supported when you import projects to a [personal namespace](../../../user/namespace/_index.md#types-of-namespaces). +When you import to a personal namespace, all contributions are assigned to +a single non-functional user called `Import User` and they cannot be reassigned. Any memberships and contributions you import are first mapped to [placeholder users](#placeholder-users). These placeholders are created on the destination instance even if @@ -183,7 +185,9 @@ A placeholder user is created for each user on the source instance, except in th - You are importing a project from [Gitea](gitea.md) and the user has been deleted on Gitea before the import. Contributions from these "ghost users" are mapped to the user who imported the project and not to a placeholder user. - You have exceeded your [placeholder user limit](#placeholder-user-limits). Contributions from any new users after exceeding your limit are - mapped to a single import user. + mapped to a single non-functional user called `Import User`. +- You are importing to a [personal namespace](../../../user/namespace/_index.md#types-of-namespaces). + Contributions are assigned to a single non-functional user called `Import User`. #### Placeholder user attributes @@ -268,7 +272,7 @@ These contributions include: You cannot determine the number of placeholder users you need in advance. When the placeholder user limit is reached, the import does not fail. -Instead, all contributions are assigned to a bot user called `Import User`. +Instead, all contributions are assigned to a single non-functional user called `Import User`. Every change creates a system note, which is not affected by the placeholder user limit. @@ -372,7 +376,6 @@ Before a user accepts the reassignment, you can [cancel the request](#cancel-rea The availability of this feature is controlled by a feature flag. For more information, see the history. -This feature is available for testing, but not ready for production use. {{< /alert >}} diff --git a/lib/api/admin/token.rb b/lib/api/admin/token.rb index cce6102da56..0f2ff0eded5 100644 --- a/lib/api/admin/token.rb +++ b/lib/api/admin/token.rb @@ -6,6 +6,8 @@ module API feature_category :system_access AUDIT_SOURCE = :api_admin_token + helpers Gitlab::InternalEventsTracking + helpers do def identify_token(plaintext) token = ::Authn::AgnosticTokenIdentifier.token_for(plaintext, AUDIT_SOURCE) @@ -13,6 +15,14 @@ module API token end + + def track_admin_api_usage_event + track_internal_event( + 'use_admin_token_api', + user: current_user, + namespace: current_user.namespace + ) + end end before do @@ -42,6 +52,8 @@ module API identified_token = identify_token(params[:token]) render_api_error!({ error: 'Not found' }, :not_found) if identified_token.revocable.nil? + track_admin_api_usage_event + status :ok present identified_token.revocable, with: identified_token.present_with, current_user: current_user @@ -69,7 +81,12 @@ module API response = identified_token.revoke!(current_user) - response.success? ? no_content! : render_api_error!({ error: response.message }, :bad_request) + if response.success? + track_admin_api_usage_event + no_content! + else + render_api_error!({ error: response.message }, :bad_request) + end end end end diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 5471cfdf194..0f68c0b2232 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -199,12 +199,8 @@ semgrep-sast: - '**/*.kt' - '**/*.properties' - '**/application*.yml' - - '**/management*.yml' - - '**/actuator*.yml' - '**/bootstrap*.yml' - '**/application*.yaml' - - '**/management*.yaml' - - '**/actuator*.yaml' - '**/bootstrap*.yaml' ## In case gitlab-advanced-sast already covers all the files that semgrep-sast would have scanned - if: $CI_COMMIT_BRANCH && @@ -241,12 +237,8 @@ semgrep-sast: - '**/*.kt' - '**/*.properties' - '**/application*.yml' - - '**/management*.yml' - - '**/actuator*.yml' - '**/bootstrap*.yml' - '**/application*.yaml' - - '**/management*.yaml' - - '**/actuator*.yaml' - '**/bootstrap*.yaml' sobelow-sast: diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml index eda68caf551..a88ce1c3556 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -252,12 +252,8 @@ semgrep-sast: - '**/*.kt' - '**/*.properties' - '**/application*.yml' - - '**/management*.yml' - - '**/actuator*.yml' - '**/bootstrap*.yml' - '**/application*.yaml' - - '**/management*.yaml' - - '**/actuator*.yaml' - '**/bootstrap*.yaml' ## In case gitlab-advanced-sast already covers all the files that semgrep-sast would have scanned - if: $CI_PIPELINE_SOURCE == "merge_request_event" && @@ -294,12 +290,8 @@ semgrep-sast: - '**/*.kt' - '**/*.properties' - '**/application*.yml' - - '**/management*.yml' - - '**/actuator*.yml' - '**/bootstrap*.yml' - '**/application*.yaml' - - '**/management*.yaml' - - '**/actuator*.yaml' - '**/bootstrap*.yaml' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. when: never @@ -328,12 +320,8 @@ semgrep-sast: - '**/*.kt' - '**/*.properties' - '**/application*.yml' - - '**/management*.yml' - - '**/actuator*.yml' - '**/bootstrap*.yml' - '**/application*.yaml' - - '**/management*.yaml' - - '**/actuator*.yaml' - '**/bootstrap*.yaml' ## In case gitlab-advanced-sast already covers all the files that semgrep-sast would have scanned - if: $CI_COMMIT_BRANCH && @@ -370,12 +358,8 @@ semgrep-sast: - '**/*.kt' - '**/*.properties' - '**/application*.yml' - - '**/management*.yml' - - '**/actuator*.yml' - '**/bootstrap*.yml' - '**/application*.yaml' - - '**/management*.yaml' - - '**/actuator*.yaml' - '**/bootstrap*.yaml' sobelow-sast: diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 520baf9f71f..e438c2ee01e 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -35883,6 +35883,9 @@ msgstr "" msgid "MemberRole|Role details" msgstr "" +msgid "MemberRole|Role is assigned to one or more admins. Remove role from all admins, then delete role." +msgstr "" + msgid "MemberRole|Role is assigned to one or more group members. Remove role from all group members, then delete role." msgstr "" diff --git a/qa/qa/page/merge_request/show.rb b/qa/qa/page/merge_request/show.rb index 6fc41c1f5ee..b4dedca1fe4 100644 --- a/qa/qa/page/merge_request/show.rb +++ b/qa/qa/page/merge_request/show.rb @@ -383,7 +383,7 @@ module QA break true unless find_element('merge-button').disabled? # If the widget shows "Merge blocked: new changes were just added" we can refresh the page and check again - next false if has_element?('head-mismatch-content', wait: 1) + next false if merge_blocked_by_new_changes? QA::Runtime::Logger.debug("MR widget text: \"#{mr_widget_text}\"") @@ -391,6 +391,11 @@ module QA end end + # Returns true when widget shows "Merge blocked: new changes were just added" + def merge_blocked_by_new_changes? + has_element?('head-mismatch-content', wait: 1) + end + def rebase! # The rebase button is disabled on load wait_until do @@ -421,6 +426,7 @@ module QA def try_to_merge!(wait_for_no_auto_merge: true) wait_until_ready_to_merge wait_until { !find_element('merge-button').text.include?('auto-merge') } if wait_for_no_auto_merge # rubocop:disable Rails/NegateInclude -- Wait for text auto-merge to change + wait_until { !merge_blocked_by_new_changes? } click_element('merge-button') end diff --git a/spec/lib/gitlab/database/sharding_key_spec.rb b/spec/lib/gitlab/database/sharding_key_spec.rb index f53e02dcfa8..038f7de3f0d 100644 --- a/spec/lib/gitlab/database/sharding_key_spec.rb +++ b/spec/lib/gitlab/database/sharding_key_spec.rb @@ -63,6 +63,7 @@ RSpec.describe 'new tables missing sharding_key', feature_category: :cell do 'ci_pipeline_schedule_variables.project_id', 'ci_build_trace_chunks.project_id', # LFK already present on p_ci_builds and cascade delete all ci resources 'p_ci_job_annotations.project_id', # LFK already present on p_ci_builds and cascade delete all ci resources + 'ci_build_pending_states.project_id', # LFK already present on p_ci_builds and cascade delete all ci resources 'ci_builds_runner_session.project_id', # LFK already present on p_ci_builds and cascade delete all ci resources 'p_ci_pipelines_config.project_id', # LFK already present on p_ci_pipelines and cascade delete all ci resources 'ci_unit_test_failures.project_id', # LFK already present on ci_unit_tests and cascade delete all ci resources diff --git a/spec/models/issue_spec.rb b/spec/models/issue_spec.rb index 22e4ee1e184..f3020ec23a3 100644 --- a/spec/models/issue_spec.rb +++ b/spec/models/issue_spec.rb @@ -474,32 +474,6 @@ RSpec.describe Issue, feature_category: :team_planning do expect(described_class.with_issue_type(%w[issue incident])) .to contain_exactly(issue, incident) end - - it 'joins the work_item_types table for filtering with issues.work_item_type_id column' do - expect do - described_class.with_issue_type([:issue, :incident]).to_a - end.to make_queries_matching( - %r{ - INNER\sJOIN\s"work_item_types"\sON\s"work_item_types"\."id"\s=\s"issues"\."work_item_type_id" - \sWHERE\s"work_item_types"\."base_type"\sIN\s\(0,\s1\) - }x - ) - end - end - - context 'when a single issue_type is provided' do - it 'uses an optimized query for a single work item type using issues.work_item_type_id column' do - expect do - described_class.with_issue_type([:incident]).to_a - end.to make_queries_matching( - %r{ - WHERE\s\("issues"\."work_item_type_id"\s= - \s\(SELECT\s"work_item_types"\."id"\sFROM\s"work_item_types" - \sWHERE\s"work_item_types"\."base_type"\s=\s1 - \sLIMIT\s1\)\) - }x - ) - end end context 'when no types are provided' do @@ -523,17 +497,6 @@ RSpec.describe Issue, feature_category: :team_planning do expect(described_class.without_issue_type(%w[issue incident])) .to contain_exactly(task) end - - it 'uses the work_item_types table and issues.work_item_type_id for filtering' do - expect do - described_class.without_issue_type(:issue).to_a - end.to make_queries_matching( - %r{ - INNER\sJOIN\s"work_item_types"\sON\s"work_item_types"\."id"\s=\s"issues"\."work_item_type_id" - \sWHERE\s"work_item_types"\."base_type"\s!=\s0 - }x - ) - end end describe '.order_severity' do diff --git a/spec/requests/api/admin/token_spec.rb b/spec/requests/api/admin/token_spec.rb index 6133c838a41..25bc08b639d 100644 --- a/spec/requests/api/admin/token_spec.rb +++ b/spec/requests/api/admin/token_spec.rb @@ -31,12 +31,32 @@ RSpec.describe API::Admin::Token, :aggregate_failures, feature_category: :system end end - let_it_be(:admin) { create(:admin) } - let_it_be(:project) { create(:project, maintainers: [admin]) } + shared_examples 'post_successful_interval_event_tracking' do + it_behaves_like 'internal event tracking' do + let(:event) { 'use_admin_token_api' } + let(:user) { api_user } + let(:namespace) { api_user.namespace } + let(:project) { nil } + subject(:track_event) { post_token } + end + end + + shared_examples 'delete_successful_interval_event_tracking' do + it_behaves_like 'internal event tracking' do + let(:event) { 'use_admin_token_api' } + let(:user) { api_user } + let(:namespace) { api_user.namespace } + let(:project) { nil } + subject(:track_event) { delete_token } + end + end + + let_it_be(:admin) { create(:admin, :with_namespace) } + let_it_be(:project) { create(:project) } let_it_be(:group) { create(:group) } let_it_be(:url) { '/admin/token' } let(:api_user) { admin } - let(:user) { create(:user) } + let_it_be(:user) { create(:user, :with_namespace) } let_it_be(:project_bot) { create(:user, :project_bot) } let_it_be(:group_bot) { create(:user, :project_bot) } @@ -69,7 +89,7 @@ RSpec.describe API::Admin::Token, :aggregate_failures, feature_category: :system [ref(:personal_access_token), lazy { personal_access_token.token }], [ref(:group_deploy_token), lazy { group_deploy_token.token }], [ref(:project_deploy_token), lazy { project_deploy_token.token }], - [ref(:user), lazy { user.feed_token }], + [ref(:user), lazy { user.reload.feed_token }], [ref(:user), lazy { user.incoming_email_token }], [ref(:oauth_application), lazy { oauth_application.plaintext_secret }], [ref(:cluster_agent_token), lazy { cluster_agent_token.token }], @@ -87,6 +107,8 @@ RSpec.describe API::Admin::Token, :aggregate_failures, feature_category: :system expect(response).to have_gitlab_http_status(:ok) expect(json_response['id']).to eq(token.id) end + + it_behaves_like 'post_successful_interval_event_tracking' end end @@ -100,6 +122,8 @@ RSpec.describe API::Admin::Token, :aggregate_failures, feature_category: :system expect(response).to have_gitlab_http_status(:ok) expect(json_response['job']['id']).to eq(ci_build.id) end + + it_behaves_like 'post_successful_interval_event_tracking' end context 'with _gitlab_session' do @@ -120,6 +144,8 @@ RSpec.describe API::Admin::Token, :aggregate_failures, feature_category: :system expect(response).to have_gitlab_http_status(:ok) expect(json_response['id']).to eq(user.id) end + + it_behaves_like 'post_successful_interval_event_tracking' end context 'with an unknown session' do @@ -166,10 +192,16 @@ RSpec.describe API::Admin::Token, :aggregate_failures, feature_category: :system expect(response).to have_gitlab_http_status(:no_content) expect(token.reload.revoked?).to be_truthy end + + it_behaves_like 'delete_successful_interval_event_tracking' end end context 'when the token can be reset' do + before do + user.reload + end + where(:token, :plaintext_attribute, :changed_attribute) do [ [ref(:user), :feed_token, :feed_token], @@ -186,6 +218,8 @@ RSpec.describe API::Admin::Token, :aggregate_failures, feature_category: :system expect(response).to have_gitlab_http_status(:no_content) end + + it_behaves_like 'delete_successful_interval_event_tracking' end end end @@ -198,6 +232,8 @@ RSpec.describe API::Admin::Token, :aggregate_failures, feature_category: :system expect(response).to have_gitlab_http_status(:no_content) end + + it_behaves_like 'delete_successful_interval_event_tracking' end context 'when the token is a ci pipeline trigger token' do diff --git a/spec/services/group_access_tokens/rotate_service_spec.rb b/spec/services/group_access_tokens/rotate_service_spec.rb index 9e25d696cd9..ee43560bd46 100644 --- a/spec/services/group_access_tokens/rotate_service_spec.rb +++ b/spec/services/group_access_tokens/rotate_service_spec.rb @@ -21,6 +21,15 @@ RSpec.describe GroupAccessTokens::RotateService, feature_category: :system_acces expect(new_token.user).to eq(token.user) expect(bot_user_membership.reload.expires_at).to be_nil end + + it_behaves_like 'internal event tracking' do + let(:event) { 'rotate_grat' } + let(:category) { described_class.name } + let(:user) { token.user } + let(:namespace) { group } + let(:project) { nil } + subject(:track_event) { response } + end end shared_examples_for 'fails to rotate the token' do diff --git a/spec/services/personal_access_tokens/rotate_service_spec.rb b/spec/services/personal_access_tokens/rotate_service_spec.rb index a4e14bc1483..75e874ded3d 100644 --- a/spec/services/personal_access_tokens/rotate_service_spec.rb +++ b/spec/services/personal_access_tokens/rotate_service_spec.rb @@ -4,7 +4,11 @@ require 'spec_helper' RSpec.describe PersonalAccessTokens::RotateService, feature_category: :system_access do describe '#execute' do - let_it_be(:token, reload: true) { create(:personal_access_token, expires_at: Time.zone.today + 30.days) } + let_it_be(:current_user) { create(:user, :with_namespace) } + let_it_be(:token, reload: true) do + create(:personal_access_token, user: current_user, expires_at: Time.zone.today + 30.days) + end + let(:params) { {} } subject(:response) { described_class.new(token.user, token, nil, params).execute } @@ -18,6 +22,7 @@ RSpec.describe PersonalAccessTokens::RotateService, feature_category: :system_ac expect(new_token.token).not_to eq(token.token) expect(new_token.expires_at).to eq(Time.zone.today + 1.week) expect(new_token.user).to eq(token.user) + expect(new_token.user.namespace).to eq(token.user.namespace) expect(new_token.organization).to eq(token.organization) expect(new_token.description).to eq(token.description) end @@ -25,6 +30,15 @@ RSpec.describe PersonalAccessTokens::RotateService, feature_category: :system_ac it_behaves_like "rotates token successfully" + it_behaves_like 'internal event tracking' do + let(:event) { 'rotate_pat' } + let(:category) { described_class.name } + let(:user) { token.user } + let(:namespace) { token.user.namespace } + let(:project) { nil } + subject(:track_event) { response } + end + it 'revokes the previous token' do expect { response }.to change { token.reload.revoked? }.from(false).to(true) diff --git a/spec/services/project_access_tokens/rotate_service_spec.rb b/spec/services/project_access_tokens/rotate_service_spec.rb index ed85da123d3..80d93b144fc 100644 --- a/spec/services/project_access_tokens/rotate_service_spec.rb +++ b/spec/services/project_access_tokens/rotate_service_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' RSpec.describe ProjectAccessTokens::RotateService, feature_category: :system_access do describe '#execute' do - let_it_be(:token, reload: true) { create(:personal_access_token) } - let(:current_user) { create(:user) } + let_it_be_with_reload(:token) { create(:personal_access_token) } + let_it_be(:current_user) { create(:user) } let(:project) { create(:project, group: create(:group)) } let(:error_message) { 'Not eligible to rotate token with access level higher than the user' } @@ -20,6 +20,14 @@ RSpec.describe ProjectAccessTokens::RotateService, feature_category: :system_acc expect(new_token.expires_at).to eq(1.week.from_now.to_date) expect(new_token.user).to eq(token.user) end + + it_behaves_like 'internal event tracking' do + let(:event) { 'rotate_prat' } + let(:category) { described_class.name } + let(:user) { token.user } + let(:namespace) { project.namespace } + subject(:track_event) { response } + end end context 'when user tries to rotate token with different access level' do