diff --git a/app/assets/stylesheets/components/related_items_list.scss b/app/assets/stylesheets/components/related_items_list.scss
index 293caf6fc87..23bd2980c48 100644
--- a/app/assets/stylesheets/components/related_items_list.scss
+++ b/app/assets/stylesheets/components/related_items_list.scss
@@ -6,6 +6,8 @@ $item-remove-button-space: 42px;
.related-items-list {
padding: $gl-padding-4;
padding-right: $gl-padding-6;
+ border-bottom-left-radius: $gl-border-size-3;
+ border-bottom-right-radius: $gl-border-size-3;
&,
.list-item:last-child {
diff --git a/app/assets/stylesheets/framework/super_sidebar.scss b/app/assets/stylesheets/framework/super_sidebar.scss
index c15bc8d9895..dd723d9f4f4 100644
--- a/app/assets/stylesheets/framework/super_sidebar.scss
+++ b/app/assets/stylesheets/framework/super_sidebar.scss
@@ -11,6 +11,18 @@
}
}
+@mixin notification-dot($color, $size, $top, $left) {
+ background-color: $color;
+ border: 2px solid $gray-10; // Same as the sidebar's background color.
+ position: absolute;
+ height: $size;
+ width: $size;
+ top: $top;
+ left: $left;
+ border-radius: 50%;
+ transition: background-color 100ms linear, border-color 100ms linear;
+}
+
.super-sidebar {
@include gl-fixed;
@include gl-top-0;
@@ -98,16 +110,12 @@
.btn-with-notification {
position: relative;
- .notification {
- background-color: $blue-500;
- border: 2px solid $gray-10; // Same as the sidebar's background color.
- position: absolute;
- height: 9px;
- width: 9px;
- top: 5px;
- left: 22px;
- border-radius: 50%;
- transition: background-color 100ms linear, border-color 100ms linear;
+ .notification-dot-info {
+ @include notification-dot($blue-500, 9px, 5px, 22px);
+ }
+
+ .notification-dot-warning {
+ @include notification-dot($orange-300, 12px, 1px, 19px);
}
&:hover,
diff --git a/app/assets/stylesheets/framework/variables.scss b/app/assets/stylesheets/framework/variables.scss
index ac27f27ae2a..e032961a253 100644
--- a/app/assets/stylesheets/framework/variables.scss
+++ b/app/assets/stylesheets/framework/variables.scss
@@ -908,6 +908,11 @@ Compare Branches
*/
$compare-branches-sticky-header-height: 68px;
+/*
+Board Swimlanes
+*/
+$board-swimlanes-headers-height: 64px;
+
/**
Bootstrap 4.2.0 introduced new icons for validating forms.
Our design system does not use those, so we are disabling them for now:
diff --git a/app/assets/stylesheets/page_bundles/merge_requests.scss b/app/assets/stylesheets/page_bundles/merge_requests.scss
index a53601445ec..f4e515704fc 100644
--- a/app/assets/stylesheets/page_bundles/merge_requests.scss
+++ b/app/assets/stylesheets/page_bundles/merge_requests.scss
@@ -819,7 +819,7 @@ $tabs-holder-z-index: 250;
.mr-widget-body,
.mr-widget-content {
- padding: $gl-padding;
+ padding: $gl-padding-12 $gl-padding;
}
.mr-widget-body-ready-merge {
@@ -840,6 +840,11 @@ $tabs-holder-z-index: 250;
}
}
+.mr-widget-grouped-section .report-block-container {
+ border-bottom-left-radius: $border-radius-default;
+ border-bottom-right-radius: $border-radius-default;
+}
+
.mr-widget-extension {
border-top: 1px solid var(--border-color, $border-color);
background-color: var(--gray-10, $gray-10);
@@ -916,7 +921,7 @@ $tabs-holder-z-index: 250;
border-left: 2px solid var(--border-color, $border-color);
position: absolute;
bottom: -17px;
- left: calc(1rem - 1px);
+ left: 26px;
height: 16px;
}
}
diff --git a/app/graphql/mutations/projects/sync_fork.rb b/app/graphql/mutations/projects/sync_fork.rb
new file mode 100644
index 00000000000..bb92f078fae
--- /dev/null
+++ b/app/graphql/mutations/projects/sync_fork.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Mutations
+ module Projects
+ class SyncFork < BaseMutation
+ graphql_name 'ProjectSyncFork'
+
+ include FindsProject
+
+ authorize :push_code
+
+ argument :project_path, GraphQL::Types::ID,
+ required: true,
+ description: 'Full path of the project to initialize.'
+
+ argument :target_branch, GraphQL::Types::String,
+ required: true,
+ description: 'Ref of the fork to fetch into.'
+
+ field :details, Types::Projects::ForkDetailsType,
+ null: true,
+ description: 'Updated fork details.'
+
+ def resolve(project_path:, target_branch:)
+ project = authorized_find!(project_path)
+ details_resolver = Resolvers::Projects::ForkDetailsResolver.new(object: project, context: context, field: nil)
+ details = details_resolver.resolve(ref: target_branch)
+
+ return respond(nil, ['This branch of this project cannot be updated from the upstream']) unless details
+
+ enqueue_sync_fork(project, target_branch, details)
+ end
+
+ def enqueue_sync_fork(project, target_branch, details)
+ return respond(details, []) if details.counts[:behind] == 0
+
+ if details.has_conflicts?
+ return respond(details, ['The synchronization cannot happen due to the merge conflict'])
+ end
+
+ return respond(details, ['This service has been called too many times.']) if rate_limit_throttled?(project)
+ return respond(details, ['Another fork sync is already in progress']) unless details.exclusive_lease.try_obtain
+
+ ::Projects::Forks::SyncWorker.perform_async(project.id, current_user.id, target_branch) # rubocop:disable CodeReuse/Worker
+
+ respond(details, [])
+ end
+
+ def rate_limit_throttled?(project)
+ Gitlab::ApplicationRateLimiter.throttled?(:project_fork_sync, scope: [project, current_user])
+ end
+
+ def respond(details, errors)
+ { details: details, errors: errors }
+ end
+ end
+ end
+end
diff --git a/app/graphql/resolvers/projects/fork_details_resolver.rb b/app/graphql/resolvers/projects/fork_details_resolver.rb
index fcc13a1bc1e..a3c60f55e14 100644
--- a/app/graphql/resolvers/projects/fork_details_resolver.rb
+++ b/app/graphql/resolvers/projects/fork_details_resolver.rb
@@ -13,8 +13,17 @@ module Resolvers
def resolve(**args)
return unless project.forked?
+ return unless authorized_fork_source?
+ return unless project.repository.branch_exists?(args[:ref])
+ return unless Feature.enabled?(:fork_divergence_counts, project)
- ::Projects::Forks::DivergenceCounts.new(project, args[:ref]).counts
+ ::Projects::Forks::Details.new(project, args[:ref])
+ end
+
+ private
+
+ def authorized_fork_source?
+ Ability.allowed?(current_user, :read_code, project.fork_source)
end
end
end
diff --git a/app/graphql/types/mutation_type.rb b/app/graphql/types/mutation_type.rb
index f72ad183fb0..d5bb35c1240 100644
--- a/app/graphql/types/mutation_type.rb
+++ b/app/graphql/types/mutation_type.rb
@@ -90,6 +90,7 @@ module Types
mount_mutation Mutations::Notes::Update::ImageDiffNote
mount_mutation Mutations::Notes::RepositionImageDiffNote
mount_mutation Mutations::Notes::Destroy
+ mount_mutation Mutations::Projects::SyncFork, calls_gitaly: true, alpha: { milestone: '15.9' }
mount_mutation Mutations::Releases::Create
mount_mutation Mutations::Releases::Update
mount_mutation Mutations::Releases::Delete
diff --git a/app/graphql/types/projects/fork_details_type.rb b/app/graphql/types/projects/fork_details_type.rb
index 88c17d89620..6157dc47255 100644
--- a/app/graphql/types/projects/fork_details_type.rb
+++ b/app/graphql/types/projects/fork_details_type.rb
@@ -9,11 +9,37 @@ module Types
field :ahead, GraphQL::Types::Int,
null: true,
+ calls_gitaly: true,
+ method: :ahead,
description: 'Number of commits ahead of upstream.'
field :behind, GraphQL::Types::Int,
null: true,
+ calls_gitaly: true,
+ method: :behind,
description: 'Number of commits behind upstream.'
+
+ field :is_syncing, GraphQL::Types::Boolean,
+ null: true,
+ method: :syncing?,
+ description: 'Indicates if there is a synchronization in progress.'
+
+ field :has_conflicts, GraphQL::Types::Boolean,
+ null: true,
+ method: :has_conflicts?,
+ description: 'Indicates if the fork conflicts with its upstream project.'
+
+ def ahead
+ counts[:ahead]
+ end
+
+ def behind
+ counts[:behind]
+ end
+
+ def counts
+ @counts ||= object.counts
+ end
end
# rubocop: enable Graphql/AuthorizeTypes
end
diff --git a/app/helpers/nav_helper.rb b/app/helpers/nav_helper.rb
index ad1aa3ad734..55a191d85b3 100644
--- a/app/helpers/nav_helper.rb
+++ b/app/helpers/nav_helper.rb
@@ -9,15 +9,27 @@ module NavHelper
header_links.include?(link)
end
+ def page_has_sidebar?
+ defined?(@left_sidebar) && @left_sidebar
+ end
+
+ def page_has_collapsed_sidebar?
+ page_has_sidebar? && collapsed_sidebar?
+ end
+
+ def page_has_collapsed_super_sidebar?
+ page_has_sidebar? && collapsed_super_sidebar?
+ end
+
def page_with_sidebar_class
class_name = page_gutter_class
if show_super_sidebar?
- class_name << 'page-with-super-sidebar' if defined?(@left_sidebar) && @left_sidebar
- class_name << 'page-with-super-sidebar-collapsed' if collapsed_super_sidebar? && @left_sidebar
+ class_name << 'page-with-super-sidebar' if page_has_sidebar?
+ class_name << 'page-with-super-sidebar-collapsed' if page_has_collapsed_super_sidebar?
else
- class_name << 'page-with-contextual-sidebar' if defined?(@left_sidebar) && @left_sidebar
- class_name << 'page-with-icon-sidebar' if collapsed_sidebar? && @left_sidebar
+ class_name << 'page-with-contextual-sidebar' if page_has_sidebar?
+ class_name << 'page-with-icon-sidebar' if page_has_collapsed_sidebar?
end
class_name -= ['right-sidebar-expanded'] if defined?(@right_sidebar) && !@right_sidebar
diff --git a/app/helpers/users/callouts_helper.rb b/app/helpers/users/callouts_helper.rb
index 2b8368dd29f..d4603afb727 100644
--- a/app/helpers/users/callouts_helper.rb
+++ b/app/helpers/users/callouts_helper.rb
@@ -11,6 +11,7 @@ module Users
UNFINISHED_TAG_CLEANUP_CALLOUT = 'unfinished_tag_cleanup_callout'
SECURITY_NEWSLETTER_CALLOUT = 'security_newsletter_callout'
MERGE_REQUEST_SETTINGS_MOVED_CALLOUT = 'merge_request_settings_moved_callout'
+ PAGES_MOVED_CALLOUT = 'pages_moved_callout'
REGISTRATION_ENABLED_CALLOUT_ALLOWED_CONTROLLER_PATHS = [/^root/, /^dashboard\S*/, /^admin\S*/].freeze
WEB_HOOK_DISABLED = 'web_hook_disabled'
ULTIMATE_FEATURE_REMOVAL_BANNER = 'ultimate_feature_removal_banner'
@@ -76,6 +77,10 @@ module Users
!user_dismissed?(MERGE_REQUEST_SETTINGS_MOVED_CALLOUT) && project.merge_requests_enabled?
end
+ def show_pages_menu_callout?
+ !user_dismissed?(PAGES_MOVED_CALLOUT)
+ end
+
def ultimate_feature_removal_banner_dismissed?(project)
return false unless project
diff --git a/app/models/projects/forks/divergence_counts.rb b/app/models/projects/forks/details.rb
similarity index 61%
rename from app/models/projects/forks/divergence_counts.rb
rename to app/models/projects/forks/details.rb
index 7d630b00083..9e09ef09022 100644
--- a/app/models/projects/forks/divergence_counts.rb
+++ b/app/models/projects/forks/details.rb
@@ -3,8 +3,11 @@
module Projects
module Forks
# Class for calculating the divergence of a fork with the source project
- class DivergenceCounts
+ class Details
+ include Gitlab::Utils::StrongMemoize
+
LATEST_COMMITS_COUNT = 10
+ LEASE_TIMEOUT = 15.minutes.to_i
EXPIRATION_TIME = 8.hours
def initialize(project, ref)
@@ -20,32 +23,55 @@ module Projects
{ ahead: ahead, behind: behind }
end
+ def exclusive_lease
+ key = ['project_details', project.id, ref].join(':')
+ uuid = Gitlab::ExclusiveLease.get_uuid(key)
+
+ Gitlab::ExclusiveLease.new(key, uuid: uuid, timeout: LEASE_TIMEOUT)
+ end
+ strong_memoize_attr :exclusive_lease
+
+ def syncing?
+ exclusive_lease.exists?
+ end
+
+ def has_conflicts?
+ !(attrs && attrs[:has_conflicts]).nil?
+ end
+
+ def update!(params)
+ Rails.cache.write(cache_key, params, expires_in: EXPIRATION_TIME)
+
+ @attrs = nil
+ end
+
private
attr_reader :project, :fork_repo, :source_repo, :ref
def cache_key
- @cache_key ||= ['project_forks', project.id, ref, 'divergence_counts']
+ @cache_key ||= ['project_fork_details', project.id, ref].join(':')
end
def divergence_counts
- fork_sha = fork_repo.commit(ref).sha
- source_sha = source_repo.commit.sha
+ sha = fork_repo.commit(ref)&.sha
+ source_sha = source_repo.commit&.sha
- cached_source_sha, cached_fork_sha, counts = Rails.cache.read(cache_key)
- return counts if cached_source_sha == source_sha && cached_fork_sha == fork_sha
+ return if sha.blank? || source_sha.blank?
- counts = calculate_divergence_counts(fork_sha, source_sha)
+ return attrs[:counts] if attrs.present? && attrs[:source_sha] == source_sha && attrs[:sha] == sha
- Rails.cache.write(cache_key, [source_sha, fork_sha, counts], expires_in: EXPIRATION_TIME)
+ counts = calculate_divergence_counts(sha, source_sha)
+
+ update!({ sha: sha, source_sha: source_sha, counts: counts })
counts
end
- def calculate_divergence_counts(fork_sha, source_sha)
+ def calculate_divergence_counts(sha, source_sha)
# If the upstream latest commit exists in the fork repo, then
# it's possible to calculate divergence counts within the fork repository.
- return fork_repo.diverging_commit_count(fork_sha, source_sha) if fork_repo.commit(source_sha)
+ return fork_repo.diverging_commit_count(sha, source_sha) if fork_repo.commit(source_sha)
# Otherwise, we need to find a commit that exists both in the fork and upstream
# in order to use this commit as a base for calculating divergence counts.
@@ -67,6 +93,10 @@ module Projects
[ahead, behind]
end
+
+ def attrs
+ @attrs ||= Rails.cache.read(cache_key)
+ end
end
end
end
diff --git a/app/services/notes/create_service.rb b/app/services/notes/create_service.rb
index c9f414f3605..8898f7feb17 100644
--- a/app/services/notes/create_service.rb
+++ b/app/services/notes/create_service.rb
@@ -162,10 +162,7 @@ module Notes
track_note_creation_usage_for_merge_requests(note) if note.for_merge_request?
track_incident_action(user, note.noteable, 'incident_comment') if note.for_issue?
track_note_creation_in_ipynb(note)
-
- if Feature.enabled?(:notes_create_service_tracking, project)
- Gitlab::Tracking.event('Notes::CreateService', 'execute', **tracking_data_for(note))
- end
+ track_note_creation_visual_review(note)
if Feature.enabled?(:route_hll_to_snowplow_phase4, project&.namespace) && note.for_commit?
metric_key_path = 'counts.commit_comment'
@@ -209,6 +206,10 @@ module Notes
Gitlab::UsageDataCounters::IpynbDiffActivityCounter.note_created(note)
end
+
+ def track_note_creation_visual_review(note)
+ Gitlab::Tracking.event('Notes::CreateService', 'execute', **tracking_data_for(note))
+ end
end
end
diff --git a/app/services/projects/forks/sync_service.rb b/app/services/projects/forks/sync_service.rb
new file mode 100644
index 00000000000..4c70d7f17f5
--- /dev/null
+++ b/app/services/projects/forks/sync_service.rb
@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Projects
+ module Forks
+ # A service for fetching upstream default branch and merging it to the fork's specified branch.
+ class SyncService < BaseService
+ ONGOING_MERGE_ERROR = 'The synchronization did not happen due to another merge in progress'
+
+ MergeError = Class.new(StandardError)
+
+ def initialize(project, user, target_branch)
+ super(project, user)
+
+ @source_project = project.fork_source
+ @head_sha = project.repository.commit(target_branch).sha
+ @target_branch = target_branch
+ @details = Projects::Forks::Details.new(project, target_branch)
+ end
+
+ def execute
+ execute_service
+
+ ServiceResponse.success
+ rescue MergeError => e
+ Gitlab::ErrorTracking.log_exception(e, { project_id: project.id, user_id: current_user.id })
+
+ ServiceResponse.error(message: e.message)
+ ensure
+ details.exclusive_lease.cancel
+ end
+
+ private
+
+ attr_reader :source_project, :head_sha, :target_branch, :details
+
+ # The method executes multiple steps:
+ #
+ # 1. Gitlab::Git::CrossRepo fetches upstream default branch into a temporary ref and returns new source sha.
+ # 2. New divergence counts are calculated using the source sha.
+ # 3. If the fork is not behind, there is nothing to merge -> exit.
+ # 4. Otherwise, continue with the new source sha.
+ # 5. If Gitlab::Git::CommandError is raised it means that merge couldn't happen due to a merge conflict. The
+ # details are updated to transfer this error to the user.
+ def execute_service
+ counts = []
+ source_sha = source_project.commit.sha
+
+ Gitlab::Git::CrossRepo.new(repository, source_project.repository)
+ .execute(source_sha) do |cross_repo_source_sha|
+ counts = repository.diverging_commit_count(head_sha, cross_repo_source_sha)
+ ahead, behind = counts
+ next if behind == 0
+
+ execute_with_fetched_source(cross_repo_source_sha, ahead)
+ end
+ rescue Gitlab::Git::CommandError => e
+ details.update!({ sha: head_sha, source_sha: source_sha, counts: counts, has_conflicts: true })
+
+ raise MergeError, e.message
+ end
+
+ def execute_with_fetched_source(cross_repo_source_sha, ahead)
+ with_linked_lfs_pointers(cross_repo_source_sha) do
+ merge_commit_id = perform_merge(cross_repo_source_sha, ahead)
+ raise MergeError, ONGOING_MERGE_ERROR unless merge_commit_id
+ end
+ end
+
+ # This method merges the upstream default branch to the fork specified branch.
+ # Depending on whether the fork branch is ahead of upstream or not, a different type of
+ # merge is performed.
+ #
+ # If the fork's branch is not ahead of the upstream (only behind), fast-forward merge is performed.
+ # However, if the fork's branch contains commits that don't exist upstream, a merge commit is created.
+ # In this case, a conflict may happen, which interrupts the merge and returns a message to the user.
+ def perform_merge(cross_repo_source_sha, ahead)
+ if ahead > 0
+ message = "Merge branch #{source_project.path}:#{source_project.default_branch} into #{target_branch}"
+
+ repository.merge_to_branch(current_user,
+ source_sha: cross_repo_source_sha,
+ target_branch: target_branch,
+ target_sha: head_sha,
+ message: message)
+ else
+ repository.ff_merge(current_user, cross_repo_source_sha, target_branch, target_sha: head_sha)
+ end
+ end
+
+ # This method links the newly merged lfs objects (if any) with the existing ones upstream.
+ # The LfsLinkService service has a limit and may raise an error if there are too many lfs objects to link.
+ # This is the reason why the block is passed:
+ #
+ # 1. Verify that there are not too many lfs objects to link
+ # 2. Execute the block (which basically performs the merge)
+ # 3. Link lfs objects
+ def with_linked_lfs_pointers(newrev, &block)
+ return yield unless project.lfs_enabled?
+
+ oldrev = head_sha
+ new_lfs_oids =
+ Gitlab::Git::LfsChanges
+ .new(repository, newrev)
+ .new_pointers(not_in: [oldrev])
+ .map(&:lfs_oid)
+
+ Projects::LfsPointers::LfsLinkService.new(project).execute(new_lfs_oids, &block)
+ rescue Projects::LfsPointers::LfsLinkService::TooManyOidsError => e
+ raise MergeError, e.message
+ end
+ end
+ end
+end
diff --git a/app/services/projects/lfs_pointers/lfs_link_service.rb b/app/services/projects/lfs_pointers/lfs_link_service.rb
index cf3cc5cd8e0..f8f03d481af 100644
--- a/app/services/projects/lfs_pointers/lfs_link_service.rb
+++ b/app/services/projects/lfs_pointers/lfs_link_service.rb
@@ -15,9 +15,9 @@ module Projects
def execute(oids)
return [] unless project&.lfs_enabled?
- if oids.size > MAX_OIDS
- raise TooManyOidsError, 'Too many LFS object ids to link, please push them manually'
- end
+ validate!(oids)
+
+ yield if block_given?
# Search and link existing LFS Object
link_existing_lfs_objects(oids)
@@ -25,6 +25,12 @@ module Projects
private
+ def validate!(oids)
+ return if oids.size <= MAX_OIDS
+
+ raise TooManyOidsError, 'Too many LFS object ids to link, please push them manually'
+ end
+
def link_existing_lfs_objects(oids)
linked_existing_objects = []
iterations = 0
diff --git a/app/views/admin/runners/register.html.haml b/app/views/admin/runners/register.html.haml
index f1477d38e98..662bb9ea00e 100644
--- a/app/views/admin/runners/register.html.haml
+++ b/app/views/admin/runners/register.html.haml
@@ -1,4 +1,7 @@
-- add_to_breadcrumbs _('Runners'), admin_runners_path
+- runner_name = "##{@runner.id} (#{@runner.short_sha})"
- breadcrumb_title s_('Runners|Register')
- page_title s_('Runners|Register'), "##{@runner.id} (#{@runner.short_sha})"
+- add_to_breadcrumbs _('Runners'), admin_runners_path
+- add_to_breadcrumbs runner_name, register_admin_runner_path(@runner)
+#js-admin-register-runner{ data: { runner_id: @runner.id, runners_path: admin_runners_path } }
diff --git a/app/views/projects/edit.html.haml b/app/views/projects/edit.html.haml
index e87005434e4..b2270e0faf7 100644
--- a/app/views/projects/edit.html.haml
+++ b/app/views/projects/edit.html.haml
@@ -7,6 +7,13 @@
= render_if_exists 'shared/ultimate_feature_removal_banner', project: @project
+- if Feature.enabled?(:show_pages_in_deployments_menu, current_user, type: :experiment)
+ = render Pajamas::AlertComponent.new(variant: :info,
+ title: _('GitLab Pages has moved'),
+ alert_options: { class: 'gl-my-5', data: { feature_id: Users::CalloutsHelper::PAGES_MOVED_CALLOUT, dismiss_endpoint: callouts_path, defer_links: 'true' } }) do |c|
+ = c.body do
+ = _('To go to GitLab Pages, on the left sidebar, select %{pages_link}.').html_safe % {pages_link: link_to('Deployments > Pages', project_pages_path(@project)).html_safe}
+
%section.settings.general-settings.no-animate.expanded#js-general-settings
.settings-header
%h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only= _('Naming, topics, avatar')
@@ -27,7 +34,6 @@
%input{ name: 'update_section', type: 'hidden', value: 'js-shared-permissions' }
%template.js-project-permissions-form-data{ type: "application/json" }= project_permissions_panel_data(@project).to_json.html_safe
.js-project-permissions-form{ data: visibility_confirm_modal_data(@project, reduce_visibility_form_id) }
-
- if show_merge_request_settings_callout?(@project)
%section.settings.expanded
= render Pajamas::AlertComponent.new(variant: :info,
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 21946c0e52b..fbb348811e0 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -3108,6 +3108,15 @@
:weight: 1
:idempotent: true
:tags: []
+- :name: projects_forks_sync
+ :worker_name: Projects::Forks::SyncWorker
+ :feature_category: :source_code_management
+ :has_external_dependencies: false
+ :urgency: :high
+ :resource_boundary: :unknown
+ :weight: 1
+ :idempotent: true
+ :tags: []
- :name: projects_git_garbage_collect
:worker_name: Projects::GitGarbageCollectWorker
:feature_category: :gitaly
diff --git a/app/workers/projects/forks/sync_worker.rb b/app/workers/projects/forks/sync_worker.rb
new file mode 100644
index 00000000000..2fa6785bc91
--- /dev/null
+++ b/app/workers/projects/forks/sync_worker.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Projects
+ module Forks
+ class SyncWorker
+ include ApplicationWorker
+
+ data_consistency :sticky
+ idempotent!
+ urgency :high
+ feature_category :source_code_management
+
+ def perform(project_id, user_id, ref)
+ project = Project.find_by_id(project_id)
+ user = User.find_by_id(user_id)
+ return unless project && user
+
+ ::Projects::Forks::SyncService.new(project, user, ref).execute
+ end
+ end
+ end
+end
diff --git a/config/feature_flags/development/notes_create_service_tracking.yml b/config/feature_flags/development/notes_create_service_tracking.yml
deleted file mode 100644
index 578c1e2a707..00000000000
--- a/config/feature_flags/development/notes_create_service_tracking.yml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-name: notes_create_service_tracking
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18890
-rollout_issue_url:
-milestone: '12.5'
-type: development
-group: group::pipeline insights
-default_enabled: false
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 17440acddc3..bbb49fde7f6 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -419,6 +419,8 @@
- 1
- - projects_finalize_project_statistics_refresh
- 1
+- - projects_forks_sync
+ - 1
- - projects_git_garbage_collect
- 1
- - projects_import_export_parallel_project_export
diff --git a/data/deprecations/14-10-old-search-migration-removal.yml b/data/deprecations/14-10-old-search-migration-removal.yml
index 1991c0ef177..4700063e68f 100644
--- a/data/deprecations/14-10-old-search-migration-removal.yml
+++ b/data/deprecations/14-10-old-search-migration-removal.yml
@@ -7,4 +7,4 @@
stage: enablement
tiers: premium, ultimate
issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/359133
- documentation_url: https://docs.gitlab.com/ee/development/elasticsearch.html#deleting-advanced-search-migrations-in-a-major-version-upgrade
+ documentation_url: https://docs.gitlab.com/ee/development/search/advanced_search_migration_styleguide.html#deleting-advanced-search-migrations-in-a-major-version-upgrade
diff --git a/data/removals/16_0/16-0-remove-embed-grafana-panels-in-markdown.yml b/data/removals/16_0/16-0-remove-embed-grafana-panels-in-markdown.yml
new file mode 100644
index 00000000000..6b6c1ec4c75
--- /dev/null
+++ b/data/removals/16_0/16-0-remove-embed-grafana-panels-in-markdown.yml
@@ -0,0 +1,15 @@
+- title: "Embedding Grafana panels in Markdown is removed"
+ announcement_milestone: "15.9"
+ announcement_date: "2023-02-22"
+ removal_milestone: "16.0"
+ removal_date: "2023-05-22"
+ breaking_change: true
+ reporter: abellucci
+ stage: monitor
+ issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/389477
+ body: |
+ The ability to add Grafana panels in GitLab Flavored Markdown is removed.
+ We intend to replace this feature with the ability to [embed charts](https://gitlab.com/groups/gitlab-org/opstrace/-/epics/33)
+ with the [GitLab Observability UI](https://gitlab.com/gitlab-org/opstrace/opstrace-ui).
+ tiers: [Free, Silver, Gold, Core, Premium, Ultimate]
+ documentation_url: https://docs.gitlab.com/ee/operations/metrics/embed_grafana.html#embed-grafana-panels-in-markdown-deprecated
diff --git a/db/post_migrate/20230216232404_add_sync_index_on_merge_request_diffs_external_diff.rb b/db/post_migrate/20230216232404_add_sync_index_on_merge_request_diffs_external_diff.rb
new file mode 100644
index 00000000000..2f9b12f89c1
--- /dev/null
+++ b/db/post_migrate/20230216232404_add_sync_index_on_merge_request_diffs_external_diff.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class AddSyncIndexOnMergeRequestDiffsExternalDiff < Gitlab::Database::Migration[2.1]
+ INDEX_NAME = 'index_merge_request_diffs_on_external_diff'
+
+ disable_ddl_transaction!
+
+ def up
+ add_concurrent_index :merge_request_diffs, :external_diff, name: INDEX_NAME
+ end
+
+ def down
+ remove_concurrent_index_by_name :merge_request_diffs, INDEX_NAME
+ end
+end
diff --git a/db/schema_migrations/20230216232404 b/db/schema_migrations/20230216232404
new file mode 100644
index 00000000000..e2fd3d7ae1f
--- /dev/null
+++ b/db/schema_migrations/20230216232404
@@ -0,0 +1 @@
+df059ad89887390a792f292b7062a2f04d901a049c2acea7b8ddaff677b8c9d5
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 5e44f1f9d4e..96f828f7453 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -30748,6 +30748,8 @@ CREATE INDEX index_merge_request_diff_details_pending_verification ON merge_requ
CREATE INDEX index_merge_request_diffs_by_id_partial ON merge_request_diffs USING btree (id) WHERE ((files_count > 0) AND ((NOT stored_externally) OR (stored_externally IS NULL)));
+CREATE INDEX index_merge_request_diffs_on_external_diff ON merge_request_diffs USING btree (external_diff);
+
CREATE INDEX index_merge_request_diffs_on_external_diff_store ON merge_request_diffs USING btree (external_diff_store);
CREATE INDEX index_merge_request_diffs_on_merge_request_id_and_id ON merge_request_diffs USING btree (merge_request_id, id);
diff --git a/doc/administration/geo/replication/configuration.md b/doc/administration/geo/replication/configuration.md
index ec17b434a84..feae2d49e8d 100644
--- a/doc/administration/geo/replication/configuration.md
+++ b/doc/administration/geo/replication/configuration.md
@@ -367,6 +367,9 @@ former is ideal for replicating data belonging to a subset of users, while the
latter is more suited to progressively rolling out Geo to a large GitLab
instance.
+NOTE:
+Geo's synchronization logic is outlined in the [documentation](../index.md). Both the solution and the documentation is subject to change from time to time. You must independently determine your legal obligations in regard to privacy and cybersecurity laws, and applicable trade control law on an ongoing basis.
+
Selective synchronization:
1. Does not restrict permissions from **secondary** sites.
diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md
index 28428f1eecc..852ac98ed10 100644
--- a/doc/administration/monitoring/prometheus/gitlab_metrics.md
+++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md
@@ -446,6 +446,7 @@ instance. For example, `cache` or `shared_state`.
| `gitlab_redis_client_exceptions_total` | Counter | 13.2 | Number of Redis client exceptions, broken down by exception class |
| `gitlab_redis_client_requests_total` | Counter | 13.2 | Number of Redis client requests |
| `gitlab_redis_client_requests_duration_seconds` | Histogram | 13.2 | Redis request latency, excluding blocking commands |
+| `gitlab_redis_client_redirections_total` | Counter | 15.10 | Number of Redis Cluster MOVED/ASK redirections, broken down by redirection type |
## Metrics shared directory
diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md
index 2af77ef5436..2cc8ebf3897 100644
--- a/doc/api/graphql/reference/index.md
+++ b/doc/api/graphql/reference/index.md
@@ -4727,6 +4727,30 @@ Input type: `ProjectSetLockedInput`
| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. |
| `project` | [`Project`](#project) | Project after mutation. |
+### `Mutation.projectSyncFork`
+
+WARNING:
+**Introduced** in 15.9.
+This feature is in Alpha. It can be changed or removed at any time.
+
+Input type: `ProjectSyncForkInput`
+
+#### Arguments
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. |
+| `projectPath` | [`ID!`](#id) | Full path of the project to initialize. |
+| `targetBranch` | [`String!`](#string) | Ref of the fork to fetch into. |
+
+#### Fields
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. |
+| `details` | [`ForkDetails`](#forkdetails) | Updated fork details. |
+| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. |
+
### `Mutation.prometheusIntegrationCreate`
Input type: `PrometheusIntegrationCreateInput`
@@ -13744,6 +13768,8 @@ Details of the fork project compared to its upstream project.
| ---- | ---- | ----------- |
| `ahead` | [`Int`](#int) | Number of commits ahead of upstream. |
| `behind` | [`Int`](#int) | Number of commits behind upstream. |
+| `hasConflicts` | [`Boolean`](#boolean) | Indicates if the fork conflicts with its upstream project. |
+| `isSyncing` | [`Boolean`](#boolean) | Indicates if there is a synchronization in progress. |
### `GeoNode`
diff --git a/doc/api/protected_tags.md b/doc/api/protected_tags.md
index 633ca441fad..d6928d7293c 100644
--- a/doc/api/protected_tags.md
+++ b/doc/api/protected_tags.md
@@ -8,17 +8,15 @@ info: To determine the technical writer assigned to the Stage/Group associated w
**Valid access levels**
-Currently, these levels are recognized:
+These access levels are recognized:
-```plaintext
-0 => No access
-30 => Developer access
-40 => Maintainer access
-```
+- `0`: No access
+- `30`: Developer role
+- `40`: Maintainer role
## List protected tags
-Gets a list of protected tags from a project.
+Gets a list of [protected tags](../user/project/protected_tags.md) from a project.
This function takes pagination parameters `page` and `per_page` to restrict the list of protected tags.
```plaintext
@@ -27,10 +25,11 @@ GET /projects/:id/protected_tags
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
-| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user |
+| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. |
```shell
-curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/protected_tags"
+curl --header "PRIVATE-TOKEN: " \
+ "https://gitlab.example.com/api/v4/projects/5/protected_tags"
```
Example response:
@@ -62,11 +61,12 @@ GET /projects/:id/protected_tags/:name
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
-| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user |
-| `name` | string | yes | The name of the tag or wildcard |
+| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. |
+| `name` | string | yes | The name of the tag or wildcard. |
```shell
-curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/protected_tags/release-1-0"
+curl --header "PRIVATE-TOKEN: " \
+ "https://gitlab.example.com/api/v4/projects/5/protected_tags/release-1-0"
```
Example response:
@@ -86,23 +86,35 @@ Example response:
## Protect repository tags
-Protects a single repository tag or several project repository
-tags using a wildcard protected tag.
+Protects a single repository tag, or several project repository
+tags, using a wildcard protected tag.
```plaintext
POST /projects/:id/protected_tags
```
```shell
-curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/protected_tags?name=*-stable&create_access_level=30"
+curl --request POST --header "PRIVATE-TOKEN: " \
+ "https://gitlab.example.com/api/v4/projects/5/protected_tags" -d '{
+ "allowed_to_create" : [
+ {
+ "user_id" : 1
+ },
+ {
+ "access_level" : 30
+ }
+ ],
+ "create_access_level" : 30,
+ "name" : "*-stable"
+}'
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
-| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user |
-| `name` | string | yes | The name of the tag or wildcard |
-| `create_access_level` | string | no | Access levels allowed to create (defaults: `40`, Maintainer role) |
-| `allowed_to_create` | array | no | Array of access levels allowed to create tags, with each described by a hash of the form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}` |
+| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. |
+| `name` | string | yes | The name of the tag or wildcard. |
+| `allowed_to_create` | array | no | Array of access levels allowed to create tags, with each described by a hash of the form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}`. |
+| `create_access_level` | string | no | Access levels allowed to create. Default: `40`, for Maintainer role. |
Example response:
@@ -128,10 +140,17 @@ DELETE /projects/:id/protected_tags/:name
```
```shell
-curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/protected_tags/*-stable"
+curl --request DELETE --header "PRIVATE-TOKEN: " \
+ "https://gitlab.example.com/api/v4/projects/5/protected_tags/*-stable"
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
-| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user |
-| `name` | string | yes | The name of the tag |
+| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. |
+| `name` | string | yes | The name of the tag. |
+
+## Related topics
+
+- [Tags API](tags.md) for all tags
+- [Tags](../user/project/repository/tags/index.md) user documentation
+- [Protected tags](../user/project/protected_tags.md) user documentation
diff --git a/doc/development/elasticsearch.md b/doc/development/advanced_search.md
similarity index 61%
rename from doc/development/elasticsearch.md
rename to doc/development/advanced_search.md
index 935964a9a90..dd05c1475ec 100644
--- a/doc/development/elasticsearch.md
+++ b/doc/development/advanced_search.md
@@ -4,16 +4,16 @@ group: Global Search
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments
---
-# Elasticsearch knowledge
+# Advanced Search development
-This area is to maintain a compendium of useful information when working with Elasticsearch.
+This page includes information about developing and working with Elasticsearch.
Information on how to enable Elasticsearch and perform the initial indexing is in
the [Elasticsearch integration documentation](../integration/advanced_search/elasticsearch.md#enable-advanced-search).
## Deep Dive
-In June 2019, Mario de la Ossa hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) on the GitLab [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) to share his domain specific knowledge with anyone who may work in this part of the codebase in the future. You can find the [recording on YouTube](https://www.youtube.com/watch?v=vrvl-tN2EaA), and the slides on [Google Slides](https://docs.google.com/presentation/d/1H-pCzI_LNrgrL5pJAIQgvLX8Ji0-jIKOg1QeJQzChug/edit) and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/c5aa32b6b07476fa8b597004899ec538/Elasticsearch_Deep_Dive.pdf). Everything covered in this deep dive was accurate as of GitLab 12.0, and while specific details may have changed since then, it should still serve as a good introduction.
+In June 2019, Mario de la Ossa hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) on the GitLab [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) to share his domain specific knowledge with anyone who may work in this part of the codebase in the future. You can find the [recording on YouTube](https://www.youtube.com/watch?v=vrvl-tN2EaA), and the slides on [Google Slides](https://docs.google.com/presentation/d/1H-pCzI_LNrgrL5pJAIQgvLX8Ji0-jIKOg1QeJQzChug/edit) and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/c5aa32b6b07476fa8b597004899ec538/Elasticsearch_Deep_Dive.pdf). Everything covered in this deep dive was accurate as of GitLab 12.0, and while specific details might have changed, it should still serve as a good introduction.
In August 2020, a second Deep Dive was hosted, focusing on [GitLab-specific architecture for multi-indices support](#zero-downtime-reindexing-with-multiple-indices). The [recording on YouTube](https://www.youtube.com/watch?v=0WdPR9oB2fg) and the [slides](https://lulalala.gitlab.io/gitlab-elasticsearch-deepdive/) are available. Everything covered in this deep dive was accurate as of GitLab 13.3.
@@ -184,305 +184,6 @@ If the current version is `v12p1`, and we need to create a new version for `v12p
1. Change the namespace for files under `v12p1` folder from `Latest` to `V12p1`
1. Make changes to files under the `latest` folder as needed
-## Creating a new Advanced Search migration
-
-> This functionality was introduced by [#234046](https://gitlab.com/gitlab-org/gitlab/-/issues/234046).
-
-NOTE:
-This only supported for indices created with GitLab 13.0 or greater.
-
-In the [`ee/elastic/migrate/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/elastic/migrate) folder, create a new file with the filename format `YYYYMMDDHHMMSS_migration_name.rb`. This format is the same for Rails database migrations.
-
-```ruby
-# frozen_string_literal: true
-
-class MigrationName < Elastic::Migration
- # Important: Any updates to the Elastic index mappings must be replicated in the respective
- # configuration files:
- # - `Elastic::Latest::Config`, for the main index.
- # - `Elastic::Latest::Config`, for standalone indices.
-
- def migrate
- end
-
- # Check if the migration has completed
- # Return true if completed, otherwise return false
- def completed?
- end
-end
-```
-
-Applied migrations are stored in `gitlab-#{RAILS_ENV}-migrations` index. All migrations not executed
-are applied by the [`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb)
-cron worker sequentially.
-
-To update Elastic index mappings, apply the configuration to the respective files:
-
-- For the main index: [`Elastic::Latest::Config`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/elastic/latest/config.rb).
-- For standalone indices: `Elastic::Latest::Config`.
-
-Migrations can be built with a retry limit and have the ability to be [failed and marked as halted](https://gitlab.com/gitlab-org/gitlab/-/blob/66e899b6637372a4faf61cfd2f254cbdd2fb9f6d/ee/lib/elastic/migration.rb#L40).
-Any data or index cleanup needed to support migration retries should be handled within the migration.
-
-### Migration helpers
-
-The following migration helpers are available in `ee/app/workers/concerns/elastic/`:
-
-#### `Elastic::MigrationBackfillHelper`
-
-Backfills a specific field in an index. In most cases, the mapping for the field should already be added.
-
-Requires the `index_name` and `field_name` methods.
-
-```ruby
-class MigrationName < Elastic::Migration
- include Elastic::MigrationBackfillHelper
-
- private
-
- def index_name
- Issue.__elasticsearch__.index_name
- end
-
- def field_name
- :schema_version
- end
-end
-```
-
-#### `Elastic::MigrationUpdateMappingsHelper`
-
-Updates a mapping in an index by calling `put_mapping` with the mapping specified.
-
-Requires the `index_name` and `new_mappings` methods.
-
-```ruby
-class MigrationName < Elastic::Migration
- include Elastic::MigrationUpdateMappingsHelper
-
- private
-
- def index_name
- Issue.__elasticsearch__.index_name
- end
-
- def new_mappings
- {
- schema_version: {
- type: 'short'
- }
- }
- end
-end
-```
-
-#### `Elastic::MigrationRemoveFieldsHelper`
-
-Removes specified fields from an index.
-
-Requires the `index_name`, `document_type` methods. If there is one field to remove, add the `field_to_remove` method, otherwise add `fields_to_remove` with an array of fields.
-
-Checks in batches if any documents that match `document_type` have the fields specified in Elasticsearch. If documents exist, uses a Painless script to perform `update_by_query`.
-
-```ruby
-class MigrationName < Elastic::Migration
- include Elastic::MigrationRemoveFieldsHelper
-
- batched!
- throttle_delay 1.minute
-
- private
-
- def index_name
- User.__elasticsearch__.index_name
- end
-
- def document_type
- 'user'
- end
-
- def fields_to_remove
- %w[two_factor_enabled has_projects]
- end
-end
-```
-
-The default batch size is `10_000`. You can override this value by specifying `BATCH_SIZE`:
-
-```ruby
-class MigrationName < Elastic::Migration
- include Elastic::MigrationRemoveFieldsHelper
-
- batched!
- BATCH_SIZE = 100
-
- ...
-end
-```
-
-#### `Elastic::MigrationObsolete`
-
-Marks a migration as obsolete when it's no longer required.
-
-```ruby
-class MigrationName < Elastic::Migration
- include Elastic::MigrationObsolete
-end
-```
-
-#### `Elastic::MigrationHelper`
-
-Contains methods you can use when a migration doesn't fit the previous examples.
-
-```ruby
-class MigrationName < Elastic::Migration
- include Elastic::MigrationHelper
-
- def migrate
- ...
- end
-
- def completed?
- ...
- end
-end
-```
-
-### Migration options supported by the `Elastic::MigrationWorker`
-
-[`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb) supports the following migration options:
-
-- `batched!` - Allow the migration to run in batches. If set, the [`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb)
-will re-enqueue itself with a delay which is set using the `throttle_delay` option described below. The batching
-must be handled within the `migrate` method, this setting controls the re-enqueuing only.
-
-- `batch_size` - Sets the number of documents modified during a `batched!` migration run. This size should be set to a value which allows the updates
-enough time to finish. This can be tuned in combination with the `throttle_delay` option described below. The batching
-must be handled within a custom `migrate` method or by using the [`Elastic::MigrationBackfillHelper`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/concerns/elastic/migration_backfill_helper.rb)
-`migrate` method which uses this setting. Default value is 1000 documents.
-
-- `throttle_delay` - Sets the wait time in between batch runs. This time should be set high enough to allow each migration batch
-enough time to finish. Additionally, the time should be less than 30 minutes since that is how often the
-[`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb)
-cron worker runs. Default value is 5 minutes.
-
-- `pause_indexing!` - Pause indexing while the migration runs. This setting will record the indexing setting before
-the migration runs and set it back to that value when the migration is completed.
-
-- `space_requirements!` - Verify that enough free space is available in the cluster when the migration runs. This setting
- will halt the migration if the storage required is not available when the migration runs. The migration must provide
- the space required in bytes by defining a `space_required_bytes` method.
-
-- `retry_on_failure` - Enable the retry on failure feature. By default, it retries
- the migration 30 times. After it runs out of retries, the migration is marked as halted.
- To customize the number of retries, pass the `max_attempts` argument:
- `retry_on_failure max_attempts: 10`
-
-```ruby
-# frozen_string_literal: true
-
-class BatchedMigrationName < Elastic::Migration
- # Declares a migration should be run in batches
- batched!
- throttle_delay 10.minutes
- pause_indexing!
- space_requirements!
- retry_on_failure
-
- # ...
-end
-```
-
-### Multi-version compatibility
-
-These Advanced Search migrations, like any other GitLab changes, need to support the case where
-[multiple versions of the application are running at the same time](multi_version_compatibility.md).
-
-Depending on the order of deployment, it's possible that the migration
-has started or finished and there's still a server running the application code from before the
-migration. We need to take this into consideration until we can
-[ensure all Advanced Search migrations start after the deployment has finished](https://gitlab.com/gitlab-org/gitlab/-/issues/321619).
-
-### Reverting a migration
-
-Because Elasticsearch does not support transactions, we always need to design our
-migrations to accommodate a situation where the application
-code is reverted after the migration has started or after it is finished.
-
-For this reason we generally defer destructive actions (for example, deletions after
-some data is moved) to a later merge request after the migrations have
-completed successfully. To be safe, for self-managed customers we should also
-defer it to another release if there is risk of important data loss.
-
-### Best practices for Advanced Search migrations
-
-Follow these best practices for best results:
-
-- When working in batches, keep the batch size under 9,000 documents
- and `throttle_delay` for at least 3 minutes. The bulk indexer is set to run
- every 1 minute and process a batch of 10,000 documents. These limits
- allow the bulk indexer time to process records before another migration
- batch is attempted.
-- To ensure that document counts are up to date, it is recommended to refresh
- the index before checking if a migration is completed.
-- Add logging statements to each migration when the migration starts, when a
- completion check occurs, and when the migration is completed. These logs
- are helpful when debugging issues with migrations.
-- Pause indexing if you're using any Elasticsearch Reindex API operations.
-- Consider adding a retry limit if there is potential for the migration to fail.
- This ensures that migrations can be halted if an issue occurs.
-
-## Deleting Advanced Search migrations in a major version upgrade
-
-Since our Advanced Search migrations usually require us to support multiple
-code paths for a long period of time, it's important to clean those up when we
-safely can.
-
-We choose to use GitLab major version upgrades as a safe time to remove
-backwards compatibility for indices that have not been fully migrated. We
-[document this in our upgrade documentation](../update/index.md#upgrading-to-a-new-major-version).
-We also choose to replace the migration code with the halted migration
-and remove tests so that:
-
-- We don't need to maintain any code that is called from our Advanced Search
- migrations.
-- We don't waste CI time running tests for migrations that we don't support
- anymore.
-- Operators who have not run this migration and who upgrade directly to the
- target version will see a message prompting them to reindex from scratch.
-
-To be extra safe, we will not delete migrations that were created in the last
-minor version before the major upgrade. So, if we are upgrading to `%14.0`,
-we should not delete migrations that were only added in `%13.12`. This is an
-extra safety net as we expect there are migrations that get merged that may
-take multiple weeks to finish on GitLab.com. It would be bad if we upgraded
-GitLab.com to `%14.0` before the migrations in `%13.12` were finished. Since
-our deployments to GitLab.com are automated and we currently don't have
-automated checks to prevent this, the extra precaution is warranted.
-Additionally, even if we did have automated checks to prevent it, we wouldn't
-actually want to hold up GitLab.com deployments on Advanced Search migrations,
-as they may still have another week to go, and that's too long to block
-deployments.
-
-### Process for removing migrations
-
-For every migration that was created 2 minor versions before the major version
-being upgraded to, we do the following:
-
-1. Confirm the migration has actually completed successfully for GitLab.com.
-1. Replace the content of the migration with:
-
- ```ruby
- include Elastic::MigrationObsolete
- ```
-
-1. Delete any spec files to support this migration.
-1. Remove any logic handling backwards compatibility for this migration. You
- can find this by looking for
- `Elastic::DataMigrationService.migration_has_finished?(:migration_name_in_lowercase)`.
-1. Create a merge request with these changes. Noting that we should not
- accidentally merge this before the major release is started.
-
## Performance Monitoring
### Prometheus
diff --git a/doc/development/changelog.md b/doc/development/changelog.md
index 27bcffe7560..5469d3fc2a8 100644
--- a/doc/development/changelog.md
+++ b/doc/development/changelog.md
@@ -94,7 +94,7 @@ EE: true
uses system fonts for all text."
- Any client-facing change to our REST and GraphQL APIs **must** have a changelog entry.
See the [complete list what comprises a GraphQL breaking change](api_graphql_styleguide.md#breaking-changes).
-- Any change that introduces an [Advanced Search migration](elasticsearch.md#creating-a-new-advanced-search-migration)
+- Any change that introduces an [Advanced Search migration](search/advanced_search_migration_styleguide.md#creating-a-new-advanced-search-migration)
**must** have a changelog entry.
- A fix for a regression introduced and then fixed in the same release (such as
fixing a bug introduced during a monthly release candidate) **should not**
diff --git a/doc/development/feature_development.md b/doc/development/feature_development.md
index e24f105250f..4c1d01e0752 100644
--- a/doc/development/feature_development.md
+++ b/doc/development/feature_development.md
@@ -79,7 +79,7 @@ Consult these topics for information on contributing to specific GitLab features
- [Adding a new Redis instance](redis/new_redis_instance.md)
- [Sidekiq guidelines](sidekiq/index.md) for working with Sidekiq workers
- [Working with Gitaly](gitaly.md)
-- [Elasticsearch integration docs](elasticsearch.md)
+- [Advanced Search integration docs](advanced_search.md)
- [Working with merge request diffs](diffs.md)
- [Approval Rules](merge_request_concepts/approval_rules.md)
- [Repository mirroring](repository_mirroring.md)
diff --git a/doc/development/search/advanced_search_migration_styleguide.md b/doc/development/search/advanced_search_migration_styleguide.md
new file mode 100644
index 00000000000..1676e666fcc
--- /dev/null
+++ b/doc/development/search/advanced_search_migration_styleguide.md
@@ -0,0 +1,306 @@
+---
+stage: Data Stores
+group: Global Search
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments
+---
+
+# Advanced Search migration style guide
+
+## Creating a new Advanced Search migration
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/234046) in GitLab 13.6.
+
+NOTE:
+This functionality is only supported for indices created in GitLab 13.0 and later.
+
+In the [`ee/elastic/migrate/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/elastic/migrate) folder, create a new file with the filename format `YYYYMMDDHHMMSS_migration_name.rb`. This format is the same for Rails database migrations.
+
+```ruby
+# frozen_string_literal: true
+
+class MigrationName < Elastic::Migration
+ # Important: Any updates to the Elastic index mappings must be replicated in the respective
+ # configuration files:
+ # - `Elastic::Latest::Config`, for the main index.
+ # - `Elastic::Latest::Config`, for standalone indices.
+
+ def migrate
+ end
+
+ # Check if the migration has completed
+ # Return true if completed, otherwise return false
+ def completed?
+ end
+end
+```
+
+Applied migrations are stored in `gitlab-#{RAILS_ENV}-migrations` index. All migrations not executed
+are applied by the [`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb)
+cron worker sequentially.
+
+To update Elastic index mappings, apply the configuration to the respective files:
+
+- For the main index: [`Elastic::Latest::Config`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/elastic/latest/config.rb).
+- For standalone indices: `Elastic::Latest::Config`.
+
+Migrations can be built with a retry limit and have the ability to be [failed and marked as halted](https://gitlab.com/gitlab-org/gitlab/-/blob/66e899b6637372a4faf61cfd2f254cbdd2fb9f6d/ee/lib/elastic/migration.rb#L40).
+Any data or index cleanup needed to support migration retries should be handled in the migration.
+
+### Migration helpers
+
+The following migration helpers are available in `ee/app/workers/concerns/elastic/`:
+
+#### `Elastic::MigrationBackfillHelper`
+
+Backfills a specific field in an index. In most cases, the mapping for the field should already be added.
+
+Requires the `index_name` and `field_name` methods.
+
+```ruby
+class MigrationName < Elastic::Migration
+ include Elastic::MigrationBackfillHelper
+
+ private
+
+ def index_name
+ Issue.__elasticsearch__.index_name
+ end
+
+ def field_name
+ :schema_version
+ end
+end
+```
+
+#### `Elastic::MigrationUpdateMappingsHelper`
+
+Updates a mapping in an index by calling `put_mapping` with the mapping specified.
+
+Requires the `index_name` and `new_mappings` methods.
+
+```ruby
+class MigrationName < Elastic::Migration
+ include Elastic::MigrationUpdateMappingsHelper
+
+ private
+
+ def index_name
+ Issue.__elasticsearch__.index_name
+ end
+
+ def new_mappings
+ {
+ schema_version: {
+ type: 'short'
+ }
+ }
+ end
+end
+```
+
+#### `Elastic::MigrationRemoveFieldsHelper`
+
+Removes specified fields from an index.
+
+Requires the `index_name`, `document_type` methods. If there is one field to remove, add the `field_to_remove` method, otherwise add `fields_to_remove` with an array of fields.
+
+Checks in batches if any documents that match `document_type` have the fields specified in Elasticsearch. If documents exist, uses a Painless script to perform `update_by_query`.
+
+```ruby
+class MigrationName < Elastic::Migration
+ include Elastic::MigrationRemoveFieldsHelper
+
+ batched!
+ throttle_delay 1.minute
+
+ private
+
+ def index_name
+ User.__elasticsearch__.index_name
+ end
+
+ def document_type
+ 'user'
+ end
+
+ def fields_to_remove
+ %w[two_factor_enabled has_projects]
+ end
+end
+```
+
+The default batch size is `10_000`. You can override this value by specifying `BATCH_SIZE`:
+
+```ruby
+class MigrationName < Elastic::Migration
+ include Elastic::MigrationRemoveFieldsHelper
+
+ batched!
+ BATCH_SIZE = 100
+
+ ...
+end
+```
+
+#### `Elastic::MigrationObsolete`
+
+Marks a migration as obsolete when it's no longer required.
+
+```ruby
+class MigrationName < Elastic::Migration
+ include Elastic::MigrationObsolete
+end
+```
+
+#### `Elastic::MigrationHelper`
+
+Contains methods you can use when a migration doesn't fit the previous examples.
+
+```ruby
+class MigrationName < Elastic::Migration
+ include Elastic::MigrationHelper
+
+ def migrate
+ ...
+ end
+
+ def completed?
+ ...
+ end
+end
+```
+
+### Migration options supported by the `Elastic::MigrationWorker`
+
+[`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb) supports the following migration options:
+
+- `batched!` - Allow the migration to run in batches. If set, [`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb)
+ re-enqueues itself with a delay which is set using the `throttle_delay` option described below. The batching
+ must be handled in the `migrate` method. This setting controls the re-enqueuing only.
+
+- `batch_size` - Sets the number of documents modified during a `batched!` migration run. This size should be set to a value which allows the updates
+ enough time to finish. This can be tuned in combination with the `throttle_delay` option described below. The batching
+ must be handled in a custom `migrate` method or by using the [`Elastic::MigrationBackfillHelper`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/concerns/elastic/migration_backfill_helper.rb)
+ `migrate` method which uses this setting. Default value is 1000 documents.
+
+- `throttle_delay` - Sets the wait time in between batch runs. This time should be set high enough to allow each migration batch
+ enough time to finish. Additionally, the time should be less than 30 minutes because that is how often the
+ [`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb)
+ cron worker runs. Default value is 5 minutes.
+
+- `pause_indexing!` - Pause indexing while the migration runs. This setting records the indexing setting before
+ the migration runs and set it back to that value when the migration is completed.
+
+- `space_requirements!` - Verify that enough free space is available in the cluster when the migration runs. This setting
+ halts the migration if the storage required is not available when the migration runs. The migration must provide
+ the space required in bytes by defining a `space_required_bytes` method.
+
+- `retry_on_failure` - Enable the retry on failure feature. By default, it retries
+ the migration 30 times. After it runs out of retries, the migration is marked as halted.
+ To customize the number of retries, pass the `max_attempts` argument:
+ `retry_on_failure max_attempts: 10`
+
+```ruby
+# frozen_string_literal: true
+
+class BatchedMigrationName < Elastic::Migration
+ # Declares a migration should be run in batches
+ batched!
+ throttle_delay 10.minutes
+ pause_indexing!
+ space_requirements!
+ retry_on_failure
+
+ # ...
+end
+```
+
+### Multi-version compatibility
+
+These Advanced Search migrations, like any other GitLab changes, need to support the case where
+[multiple versions of the application are running at the same time](../multi_version_compatibility.md).
+
+Depending on the order of deployment, it's possible that the migration
+has started or finished and there's still a server running the application code from before the
+migration. We need to take this into consideration until we can
+[ensure all Advanced Search migrations start after the deployment has finished](https://gitlab.com/gitlab-org/gitlab/-/issues/321619).
+
+### Reverting a migration
+
+Because Elasticsearch does not support transactions, we always need to design our
+migrations to accommodate a situation where the application
+code is reverted after the migration has started or after it is finished.
+
+For this reason we generally defer destructive actions (for example, deletions after
+some data is moved) to a later merge request after the migrations have
+completed successfully. To be safe, for self-managed customers we should also
+defer it to another release if there is risk of important data loss.
+
+### Best practices for Advanced Search migrations
+
+Follow these best practices for best results:
+
+- When working in batches, keep the batch size under 9,000 documents
+ and `throttle_delay` for at least 3 minutes. The bulk indexer is set to run
+ every 1 minute and process a batch of 10,000 documents. These limits
+ allow the bulk indexer time to process records before another migration
+ batch is attempted.
+- To ensure that document counts are up to date, you should refresh
+ the index before checking if a migration is completed.
+- Add logging statements to each migration when the migration starts, when a
+ completion check occurs, and when the migration is completed. These logs
+ are helpful when debugging issues with migrations.
+- Pause indexing if you're using any Elasticsearch Reindex API operations.
+- Consider adding a retry limit if there is potential for the migration to fail.
+ This ensures that migrations can be halted if an issue occurs.
+
+## Deleting Advanced Search migrations in a major version upgrade
+
+Because our Advanced Search migrations usually require us to support multiple
+code paths for a long period of time, it's important to clean those up when we
+safely can.
+
+We choose to use GitLab major version upgrades as a safe time to remove
+backwards compatibility for indices that have not been fully migrated. We
+[document this in our upgrade documentation](../../update/index.md#upgrading-to-a-new-major-version).
+We also choose to replace the migration code with the halted migration
+and remove tests so that:
+
+- We don't need to maintain any code that is called from our Advanced Search
+ migrations.
+- We don't waste CI time running tests for migrations that we don't support
+ anymore.
+- Operators who have not run this migration and who upgrade directly to the
+ target version see a message prompting them to reindex from scratch.
+
+To be extra safe, we do not delete migrations that were created in the last
+minor version before the major upgrade. So, if we are upgrading to `%14.0`,
+we should not delete migrations that were only added in `%13.12`. This
+extra safety net allows for migrations that might
+take multiple weeks to finish on GitLab.com. It would be bad if we upgraded
+GitLab.com to `%14.0` before the migrations in `%13.12` were finished. Because
+our deployments to GitLab.com are automated and we don't have
+automated checks to prevent this, the extra precaution is warranted.
+Additionally, even if we did have automated checks to prevent it, we wouldn't
+actually want to hold up GitLab.com deployments on Advanced Search migrations,
+as they may still have another week to go, and that's too long to block
+deployments.
+
+### Process for removing migrations
+
+For every migration that was created 2 minor versions before the major version
+being upgraded to, we do the following:
+
+1. Confirm the migration has actually completed successfully for GitLab.com.
+1. Replace the content of the migration with:
+
+ ```ruby
+ include Elastic::MigrationObsolete
+ ```
+
+1. Delete any spec files to support this migration.
+1. Remove any logic handling backwards compatibility for this migration. You
+ can find this by looking for
+ `Elastic::DataMigrationService.migration_has_finished?(:migration_name_in_lowercase)`.
+1. Create a merge request with these changes. Noting that we should not
+ accidentally merge this before the major release is started.
diff --git a/doc/integration/jira/troubleshooting.md b/doc/integration/jira/troubleshooting.md
index 0e679693614..680edf616ab 100644
--- a/doc/integration/jira/troubleshooting.md
+++ b/doc/integration/jira/troubleshooting.md
@@ -98,9 +98,9 @@ p.each do |project|
end
```
-## `500 Whoops` when accessing a Jira issue in GitLab
+## `500 We're sorry` when accessing a Jira issue in GitLab
-When accessing a Jira issue in GitLab, you might get a `500 Whoops, something went wrong on our end` error.
+When accessing a Jira issue in GitLab, you might get a `500 We're sorry. Something went wrong on our end` error.
Check [`production.log`](../../administration/logs/index.md#productionlog) to see if it contains the following exception:
```plaintext
diff --git a/doc/update/removals.md b/doc/update/removals.md
index 110a3ee29e8..a83b10142a9 100644
--- a/doc/update/removals.md
+++ b/doc/update/removals.md
@@ -34,6 +34,18 @@ For removal reviewers (Technical Writers only):
https://about.gitlab.com/handbook/marketing/blog/release-posts/#update-the-removals-doc
-->
+## Removed in 16.0
+
+### Embedding Grafana panels in Markdown is removed
+
+WARNING:
+This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/).
+Review the details carefully before upgrading.
+
+The ability to add Grafana panels in GitLab Flavored Markdown is removed.
+We intend to replace this feature with the ability to [embed charts](https://gitlab.com/groups/gitlab-org/opstrace/-/epics/33)
+with the [GitLab Observability UI](https://gitlab.com/gitlab-org/opstrace/opstrace-ui).
+
## Removed in 15.9
### Live Preview no longer available in the Web IDE
diff --git a/doc/user/group/import/index.md b/doc/user/group/import/index.md
index 9190006fd67..6bb9f402d97 100644
--- a/doc/user/group/import/index.md
+++ b/doc/user/group/import/index.md
@@ -173,10 +173,22 @@ To view group import history:
### Migrated group items
-The [`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/group/import_export.yml)
-file for groups lists many of the items imported when migrating groups by direct transfer. View this file in the branch
-for your version of GitLab to see the list of items relevant to you. For example,
-[`import_export.yml` on the `14-10-stable-ee` branch](https://gitlab.com/gitlab-org/gitlab/-/blob/14-10-stable-ee/lib/gitlab/import_export/group/import_export.yml).
+The group items that are migrated depend on the version of GitLab you use on the destination. To determine if a
+specific group item is migrated:
+
+1. Check the [`groups/stage.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/bulk_imports/groups/stage.rb)
+ file for all editions and the
+ [`groups/stage.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/ee/bulk_imports/groups/stage.rb) file
+ for Enterprise Edition for your version on the destination. For example, for version 15.9:
+ - (all editions).
+ - (Enterprise
+ Edition).
+1. Check the
+ [`group/import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/group/import_export.yml)
+ file for groups for your version on the destination. For example, for version 15.9:
+ .
+
+Any other group items are **not** migrated.
Group items that are migrated to the destination GitLab instance include:
@@ -203,8 +215,6 @@ Group items that are migrated to the destination GitLab instance include:
- Subgroups ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18938) in GitLab 13.7)
- Uploads ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18938) in GitLab 13.7)
-Any other items are **not** migrated.
-
### Migrated project items (beta)
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) in GitLab 14.4 [with a flag](../../feature_flags.md) named `bulk_import_projects`. Disabled by default.
@@ -215,10 +225,22 @@ On self-managed GitLab, migrating project resources when migrating groups is not
To make it available ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named
`bulk_import_projects`. On GitLab.com, groups are migrated with all their projects by default.
-The [`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/project/import_export.yml)
-file for projects lists many of the items imported when migrating projects using group migration. View this file in the branch
-for your version of GitLab to see the list of items relevant to you. For example,
-[`import_export.yml` on the `14-10-stable-ee` branch](https://gitlab.com/gitlab-org/gitlab/-/blob/14-10-stable-ee/lib/gitlab/import_export/project/import_export.yml).
+The project items that are migrated depends on the version of GitLab you use on the destination. To determine if a
+specific project item is migrated:
+
+1. Check the [`projects/stage.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/bulk_imports/projects/stage.rb)
+ file for all editions and the
+ [`projects/stage.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/ee/bulk_imports/projects/stage.rb)
+ file for Enterprise Edition for your version on the destination. For example, for version 15.9:
+ - (all editions).
+ - (Enterprise
+ Edition).
+1. Check the
+ [`project/import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/project/import_export.yml)
+ file for projects for your version on the destination. For example, for version 15.9:
+ .
+
+Any other project items are **not** migrated.
WARNING:
Migrating projects when migrating groups by direct transfer is in [Beta](../../../policy/alpha-beta-support.md#beta-features)
@@ -390,7 +412,7 @@ For example:
The [`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/group/import_export.yml)
file for groups lists items exported and imported when migrating groups using file exports. View this file in the branch
-for your version of GitLab to see the list of items relevant to you. For example,
+for your version of GitLab to check which items can be imported to the destination GitLab instance. For example,
[`import_export.yml` on the `14-10-stable-ee` branch](https://gitlab.com/gitlab-org/gitlab/-/blob/14-10-stable-ee/lib/gitlab/import_export/group/import_export.yml).
Group items that are exported include:
diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb
index 8c1cd7c21c2..71629eb701c 100644
--- a/lib/gitlab/application_rate_limiter.rb
+++ b/lib/gitlab/application_rate_limiter.rb
@@ -55,6 +55,7 @@ module Gitlab
phone_verification_verify_code: { threshold: 10, interval: 10.minutes },
namespace_exists: { threshold: 20, interval: 1.minute },
fetch_google_ip_list: { threshold: 10, interval: 1.minute },
+ project_fork_sync: { threshold: 10, interval: 30.minutes },
jobs_index: { threshold: 600, interval: 1.minute },
bulk_import: { threshold: 6, interval: 1.minute },
projects_api_rate_limit_unauthenticated: {
diff --git a/lib/gitlab/ci/components/header.rb b/lib/gitlab/ci/components/header.rb
new file mode 100644
index 00000000000..732874d7a88
--- /dev/null
+++ b/lib/gitlab/ci/components/header.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Components
+ ##
+ # Components::Header class represents full component specification that is being prepended as first YAML document
+ # in the CI Component file.
+ #
+ class Header
+ attr_reader :errors
+
+ def initialize(header)
+ @header = header
+ @errors = []
+ end
+
+ def empty?
+ inputs_spec.to_h.empty?
+ end
+
+ def inputs(args)
+ @input ||= Ci::Input::Inputs.new(inputs_spec, args)
+ end
+
+ def context(args)
+ inputs(args).then do |input|
+ raise ArgumentError unless input.valid?
+
+ Ci::Interpolation::Context.new({ inputs: input.to_hash })
+ end
+ end
+
+ private
+
+ def inputs_spec
+ @header.dig(:spec, :inputs)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/input/arguments/base.rb b/lib/gitlab/ci/input/arguments/base.rb
new file mode 100644
index 00000000000..a46037c40ce
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/base.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Input
+ module Arguments
+ ##
+ # Input::Arguments::Base is a common abstraction for input arguments:
+ # - required
+ # - optional
+ # - with a default value
+ #
+ class Base
+ attr_reader :key, :value, :spec, :errors
+
+ ArgumentNotValidError = Class.new(StandardError)
+
+ def initialize(key, spec, value)
+ @key = key # hash key / argument name
+ @value = value # user-provided value
+ @spec = spec # configured specification
+ @errors = []
+
+ unless value.is_a?(String) || value.nil? # rubocop:disable Style/IfUnlessModifier
+ @errors.push("unsupported value in input argument `#{key}`")
+ end
+
+ validate!
+ end
+
+ def valid?
+ @errors.none?
+ end
+
+ def validate!
+ raise NotImplementedError
+ end
+
+ def to_value
+ raise NotImplementedError
+ end
+
+ def to_hash
+ raise ArgumentNotValidError unless valid?
+
+ @output ||= { key => to_value }
+ end
+
+ def self.matches?(spec)
+ raise NotImplementedError
+ end
+
+ private
+
+ def error(message)
+ @errors.push("`#{@key}` input: #{message}")
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/input/arguments/default.rb b/lib/gitlab/ci/input/arguments/default.rb
new file mode 100644
index 00000000000..fd61c1ab786
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/default.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Input
+ module Arguments
+ ##
+ # Input::Arguments::Default class represents user-provided input argument that has a default value.
+ #
+ class Default < Input::Arguments::Base
+ def validate!
+ error('invalid specification') unless default.present?
+ end
+
+ ##
+ # User-provided value needs to be specified, but it may be an empty string:
+ #
+ # ```yaml
+ # inputs:
+ # env:
+ # default: development
+ #
+ # with:
+ # env: ""
+ # ```
+ #
+ # The configuration above will result in `env` being an empty string.
+ #
+ def to_value
+ value.nil? ? default : value
+ end
+
+ def default
+ spec[:default]
+ end
+
+ def self.matches?(spec)
+ spec.count == 1 && spec.each_key.first == :default
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/input/arguments/options.rb b/lib/gitlab/ci/input/arguments/options.rb
new file mode 100644
index 00000000000..debc89b10bd
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/options.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Input
+ module Arguments
+ ##
+ # Input::Arguments::Options class represents user-provided input argument that is an enum, and is only valid
+ # when the value provided is listed as an acceptable one.
+ #
+ class Options < Input::Arguments::Base
+ ##
+ # An empty value is valid if it is allowlisted:
+ #
+ # ```yaml
+ # inputs:
+ # run:
+ # - ""
+ # - tests
+ #
+ # with:
+ # run: ""
+ # ```
+ #
+ # The configuration above will return an empty value.
+ #
+ def validate!
+ return error('argument specification invalid') if options.to_a.empty?
+
+ if !value.nil?
+ error("argument value #{value} not allowlisted") unless options.include?(value)
+ else
+ error('argument not provided')
+ end
+ end
+
+ def to_value
+ value
+ end
+
+ def options
+ spec[:options]
+ end
+
+ def self.matches?(spec)
+ spec.count == 1 && spec.each_key.first == :options
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/input/arguments/required.rb b/lib/gitlab/ci/input/arguments/required.rb
new file mode 100644
index 00000000000..b4e218ed29e
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/required.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Input
+ module Arguments
+ ##
+ # Input::Arguments::Required class represents user-provided required input argument.
+ #
+ class Required < Input::Arguments::Base
+ ##
+ # The value has to be defined, but it may be empty.
+ #
+ def validate!
+ error('required value has not been provided') if value.nil?
+ end
+
+ def to_value
+ value
+ end
+
+ ##
+ # Required arguments do not have nested configuration. It has to be defined a null value.
+ #
+ # ```yaml
+ # spec:
+ # inputs:
+ # website:
+ # ```
+ #
+ # An empty value, that has no specification is also considered as a "required" input, however we should
+ # never see that being used, because it will be rejected by Ci::Config::Header validation.
+ #
+ # ```yaml
+ # spec:
+ # inputs:
+ # website: ""
+ # ```
+ def self.matches?(spec)
+ spec.to_s.empty?
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/input/arguments/unknown.rb b/lib/gitlab/ci/input/arguments/unknown.rb
new file mode 100644
index 00000000000..5873e6e66a6
--- /dev/null
+++ b/lib/gitlab/ci/input/arguments/unknown.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Input
+ module Arguments
+ ##
+ # Input::Arguments::Unknown object gets fabricated when we can't match an input argument entry with any known
+ # specification. It is matched as the last one, and always returns an error.
+ #
+ class Unknown < Input::Arguments::Base
+ def validate!
+ if spec.is_a?(Hash) && spec.count == 1
+ error("unrecognized input argument specification: `#{spec.each_key.first}`")
+ else
+ error('unrecognized input argument definition')
+ end
+ end
+
+ def to_value
+ raise ArgumentError, 'unknown argument value'
+ end
+
+ def self.matches?(*)
+ true
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/input/inputs.rb b/lib/gitlab/ci/input/inputs.rb
new file mode 100644
index 00000000000..743ae2ecf1e
--- /dev/null
+++ b/lib/gitlab/ci/input/inputs.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Input
+ ##
+ # Inputs::Input class represents user-provided inputs, configured using `with:` keyword.
+ #
+ # Input arguments are only valid with an associated component's inputs specification from component's header.
+ #
+ class Inputs
+ UnknownSpecArgumentError = Class.new(StandardError)
+
+ ARGUMENTS = [
+ Input::Arguments::Required, # Input argument is required
+ Input::Arguments::Default, # Input argument has a default value
+ Input::Arguments::Options, # Input argument that needs to be allowlisted
+ Input::Arguments::Unknown # Input argument has not been recognized
+ ].freeze
+
+ def initialize(spec, args)
+ @spec = spec
+ @args = args
+ @inputs = []
+ @errors = []
+
+ validate!
+ fabricate!
+ end
+
+ def errors
+ @errors + @inputs.flat_map(&:errors)
+ end
+
+ def valid?
+ errors.none?
+ end
+
+ def unknown
+ @args.keys - @spec.keys
+ end
+
+ def count
+ @inputs.count
+ end
+
+ def to_hash
+ @inputs.inject({}) do |hash, argument|
+ raise ArgumentError unless argument.valid?
+
+ hash.merge(argument.to_hash)
+ end
+ end
+
+ private
+
+ def validate!
+ @errors.push("unknown input arguments: #{unknown.inspect}") if unknown.any?
+ end
+
+ def fabricate!
+ @spec.each do |key, spec|
+ argument = ARGUMENTS.find { |klass| klass.matches?(spec) }
+
+ raise UnknownSpecArgumentError if argument.nil?
+
+ @inputs.push(argument.new(key, spec, @args[key]))
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb
index 3ee8768d509..00a7387afe2 100644
--- a/lib/gitlab/instrumentation/redis_base.rb
+++ b/lib/gitlab/instrumentation/redis_base.rb
@@ -118,6 +118,14 @@ module Gitlab
@exception_counter.increment({ storage: storage_key, exception: ex.class.to_s })
end
+ def instance_count_cluster_redirection(ex)
+ # This metric is meant to give a client side view of how often are commands
+ # redirected to the right node, especially during resharding..
+ # This metric can be used for Redis alerting and service health monitoring.
+ @redirection_counter ||= Gitlab::Metrics.counter(:gitlab_redis_client_redirections_total, 'Client side Redis Cluster redirection count, per Redis node, per slot')
+ @redirection_counter.increment(decompose_redirection_message(ex.message).merge({ storage: storage_key }))
+ end
+
def instance_observe_duration(duration)
@request_latency_histogram ||= Gitlab::Metrics.histogram(
:gitlab_redis_client_requests_duration_seconds,
@@ -166,6 +174,11 @@ module Gitlab
def build_key(namespace)
"#{storage_key}_#{namespace}"
end
+
+ def decompose_redirection_message(err_msg)
+ redirection_type, _, target_node_key = err_msg.split
+ { redirection_type: redirection_type, target_node_key: target_node_key }
+ end
end
end
end
diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb
index 82531883810..2a86b9e4202 100644
--- a/lib/gitlab/instrumentation/redis_interceptor.rb
+++ b/lib/gitlab/instrumentation/redis_interceptor.rb
@@ -40,7 +40,12 @@ module Gitlab
yield
rescue ::Redis::BaseError => ex
- instrumentation_class.instance_count_exception(ex)
+ if ex.message.start_with?('MOVED', 'ASK')
+ instrumentation_class.instance_count_cluster_redirection(ex)
+ else
+ instrumentation_class.instance_count_exception(ex)
+ end
+
instrumentation_class.log_exception(ex)
raise ex
ensure
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 2a02f76fa64..aac81ca574e 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -4743,6 +4743,9 @@ msgstr ""
msgid "Analytics|Analytics dashboards"
msgstr ""
+msgid "Analytics|Dashboards are created by editing the projects dashboard files."
+msgstr ""
+
msgid "Analyze your dependencies for known vulnerabilities."
msgstr ""
@@ -19001,6 +19004,9 @@ msgstr ""
msgid "GitLab Pages"
msgstr ""
+msgid "GitLab Pages has moved"
+msgstr ""
+
msgid "GitLab Shell"
msgstr ""
@@ -32589,9 +32595,6 @@ msgstr ""
msgid "ProductAnalytics|An error occurred while fetching data. Refresh the page to try again."
msgstr ""
-msgid "ProductAnalytics|Analytics dashboards"
-msgstr ""
-
msgid "ProductAnalytics|Analyze your product with Product Analytics"
msgstr ""
@@ -32652,9 +32655,6 @@ msgstr ""
msgid "ProductAnalytics|Creating your product analytics instance..."
msgstr ""
-msgid "ProductAnalytics|Dashboards are created by editing the projects dashboard files."
-msgstr ""
-
msgid "ProductAnalytics|Data"
msgstr ""
@@ -37100,6 +37100,9 @@ msgstr ""
msgid "Runners|Checkbox"
msgstr ""
+msgid "Runners|Choose an executor when prompted by the command line. Executors run builds in different environments. %{linkStart}Not sure which one to select?%{linkEnd}"
+msgstr ""
+
msgid "Runners|Choose your preferred GitLab Runner"
msgstr ""
@@ -37118,6 +37121,9 @@ msgstr ""
msgid "Runners|Containers"
msgstr ""
+msgid "Runners|Copy and paste the following command into your command line to register the runner."
+msgstr ""
+
msgid "Runners|Copy instructions"
msgstr ""
@@ -37192,6 +37198,12 @@ msgstr ""
msgid "Runners|Get started with runners"
msgstr ""
+msgid "Runners|GitLab Runner must be installed before you can register a runner. %{linkStart}How do I install GitLab Runner?%{linkEnd}"
+msgstr ""
+
+msgid "Runners|Go to runners page"
+msgstr ""
+
msgid "Runners|Group"
msgstr ""
@@ -37237,6 +37249,9 @@ msgstr ""
msgid "Runners|Maintenance note"
msgstr ""
+msgid "Runners|Manually verify that the runner is available to pick up jobs."
+msgstr ""
+
msgid "Runners|Maximum amount of time the runner can run before it terminates. If a project has a shorter job timeout period, the job timeout period of the instance runner is used instead."
msgstr ""
@@ -37309,6 +37324,9 @@ msgstr ""
msgid "Runners|Operating systems"
msgstr ""
+msgid "Runners|Optional. Step 3"
+msgstr ""
+
msgid "Runners|Owner"
msgstr ""
@@ -37344,6 +37362,9 @@ msgstr ""
msgid "Runners|Register"
msgstr ""
+msgid "Runners|Register \"%{runnerDescription}\" runner"
+msgstr ""
+
msgid "Runners|Register a group runner"
msgstr ""
@@ -37359,6 +37380,9 @@ msgstr ""
msgid "Runners|Register as many runners as you want. You can register runners as separate users, on separate servers, and on your local machine."
msgstr ""
+msgid "Runners|Register runner"
+msgstr ""
+
msgid "Runners|Registration token"
msgstr ""
@@ -37515,6 +37539,12 @@ msgstr ""
msgid "Runners|Status"
msgstr ""
+msgid "Runners|Step 1"
+msgstr ""
+
+msgid "Runners|Step 2"
+msgstr ""
+
msgid "Runners|Stop the runner from accepting new jobs."
msgstr ""
@@ -37524,6 +37554,9 @@ msgstr ""
msgid "Runners|Tags control which type of jobs a runner can handle. By tagging a runner, you make sure shared runners only handle the jobs they are equipped to run."
msgstr ""
+msgid "Runners|The %{boldStart}runner token%{boldEnd} %{token} displays %{boldStart}only for a short time%{boldEnd}, and is stored in the %{codeStart}config.toml%{codeEnd} after you create the runner. It will not be visible once the runner is registered."
+msgstr ""
+
msgid "Runners|The project, group or instance where the runner was registered. Instance runners are always owned by Administrator."
msgstr ""
@@ -37541,6 +37574,9 @@ msgstr[1] ""
msgid "Runners|This group currently has no stale runners."
msgstr ""
+msgid "Runners|This may not be needed if you manage your runner as a %{linkStart}system or user service%{linkEnd}."
+msgstr ""
+
msgid "Runners|This runner has not run any jobs."
msgstr ""
@@ -44913,6 +44949,9 @@ msgstr ""
msgid "To get started, use the link below to confirm your account."
msgstr ""
+msgid "To go to GitLab Pages, on the left sidebar, select %{pages_link}."
+msgstr ""
+
msgid "To help improve GitLab, we would like to periodically %{docs_link}. This can be changed at any time in %{settings_link}."
msgstr ""
diff --git a/package.json b/package.json
index e5081047ee8..648474d49bb 100644
--- a/package.json
+++ b/package.json
@@ -56,7 +56,7 @@
"@gitlab/favicon-overlay": "2.0.0",
"@gitlab/fonts": "^1.2.0",
"@gitlab/svgs": "3.21.0",
- "@gitlab/ui": "56.1.2",
+ "@gitlab/ui": "56.2.0",
"@gitlab/visual-review-tools": "1.7.3",
"@gitlab/web-ide": "0.0.1-dev-20230223005157",
"@rails/actioncable": "6.1.4-7",
diff --git a/public/500.html b/public/500.html
index b9de6994d58..9c04a3db339 100644
--- a/public/500.html
+++ b/public/500.html
@@ -73,7 +73,7 @@
500
-
Whoops, something went wrong on our end.
+
We're sorry. Something went wrong on our end.
Try refreshing the page, or going back and attempting the action again.