Add latest changes from gitlab-org/gitlab@master

2022-02-08 15:12:33 +00:00 · 2022-02-08 15:12:33 +00:00 · 6225d57e55
parent dcc096f219
commit 6225d57e55
39 changed files with 387 additions and 162 deletions
--- a/app/controllers/admin/application_settings_controller.rb
+++ b/app/controllers/admin/application_settings_controller.rb
@ -62,11 +62,11 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
  def usage_data
    respond_to do |format|
      format.html do
-        usage_data_json = Gitlab::Json.pretty_generate(Gitlab::Usage::ServicePingReport.for(mode: :values, cached: true))
+        usage_data_json = Gitlab::Json.pretty_generate(Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values, cached: true))

        render html: Gitlab::Highlight.highlight('payload.json', usage_data_json, language: 'json')
      end
-      format.json { render json: Gitlab::Usage::ServicePingReport.for(mode: :values, cached: true).to_json }
+      format.json { render json: Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values, cached: true).to_json }
    end
  end

--- a/app/controllers/admin/instance_review_controller.rb
+++ b/app/controllers/admin/instance_review_controller.rb
@ -16,7 +16,7 @@ class Admin::InstanceReviewController < Admin::ApplicationController
    }

    if Gitlab::CurrentSettings.usage_ping_enabled?
-      data = Gitlab::Usage::ServicePingReport.for(mode: :values, cached: true)
+      data = Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values, cached: true)
      counts = data[:counts]

      result[:instance_review].merge!(
--- a/app/models/personal_access_token.rb
+++ b/app/models/personal_access_token.rb
@ -33,6 +33,7 @@ class PersonalAccessToken < ApplicationRecord
  scope :preload_users, -> { preload(:user) }
  scope :order_expires_at_asc, -> { reorder(expires_at: :asc) }
  scope :order_expires_at_desc, -> { reorder(expires_at: :desc) }
+  scope :project_access_token, -> { includes(:user).where(user: { user_type: :project_bot }) }

  validates :scopes, presence: true
  validate :validate_scopes
@ -93,6 +94,10 @@ class PersonalAccessToken < ApplicationRecord
    "#{self.class.token_prefix}#{token}"
  end

+  def project_access_token?
+    user&.project_bot?
+  end
+
  protected

  def validate_scopes
--- a/app/services/alert_management/create_alert_issue_service.rb
+++ b/app/services/alert_management/create_alert_issue_service.rb
@ -22,8 +22,6 @@ module AlertManagement
      return result unless result.success?

      issue = result.payload[:issue]
-      return error(object_errors(alert), issue) unless associate_alert_with_issue(issue)
-
      update_title_for(issue)

      SystemNoteService.new_alert_issue(alert, issue, user)
@ -47,14 +45,11 @@ module AlertManagement
        user,
        title: alert_presenter.title,
        description: alert_presenter.issue_description,
-        severity: alert.severity
+        severity: alert.severity,
+        alert: alert
      ).execute
    end

-    def associate_alert_with_issue(issue)
-      alert.update(issue_id: issue.id)
-    end
-
    def update_title_for(issue)
      return unless issue.title == DEFAULT_ALERT_TITLE

@ -78,9 +73,5 @@ module AlertManagement
        alert.present
      end
    end
-
-    def object_errors(object)
-      object.errors.full_messages.to_sentence
-    end
  end
 end
--- a/app/services/incident_management/incidents/create_service.rb
+++ b/app/services/incident_management/incidents/create_service.rb
@ -2,15 +2,16 @@

 module IncidentManagement
  module Incidents
-    class CreateService < BaseService
+    class CreateService < ::BaseProjectService
      ISSUE_TYPE = 'incident'

-      def initialize(project, current_user, title:, description:, severity: IssuableSeverity::DEFAULT)
-        super(project, current_user)
+      def initialize(project, current_user, title:, description:, severity: IssuableSeverity::DEFAULT, alert: nil)
+        super(project: project, current_user: current_user)

        @title = title
        @description = description
        @severity = severity
+        @alert = alert
      end

      def execute
@ -21,11 +22,16 @@ module IncidentManagement
            title: title,
            description: description,
            issue_type: ISSUE_TYPE,
-            severity: severity
+            severity: severity,
+            alert_management_alert: alert
          },
          spam_params: nil
        ).execute

+        if alert
+          return error(alert.errors.full_messages.to_sentence, issue) unless alert.valid?
+        end
+
        return error(issue.errors.full_messages.to_sentence, issue) unless issue.valid?

        success(issue)
@ -33,7 +39,7 @@ module IncidentManagement

      private

-      attr_reader :title, :description, :severity
+      attr_reader :title, :description, :severity, :alert

      def success(issue)
        ServiceResponse.success(payload: { issue: issue })
--- a/app/services/projects/overwrite_project_service.rb
+++ b/app/services/projects/overwrite_project_service.rb
@ -6,30 +6,34 @@ module Projects
      return unless source_project && source_project.namespace_id == @project.namespace_id

      start_time = ::Gitlab::Metrics::System.monotonic_time
+      original_source_name = source_project.name
+      original_source_path = source_project.path
+      tmp_source_name, tmp_source_path = tmp_source_project_name(source_project)

-      Project.transaction do
-        move_before_destroy_relationships(source_project)
-        # Reset is required in order to get the proper
-        # uncached fork network method calls value.
-        ::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/340256') do
-          destroy_old_project(source_project.reset)
-        end
-        rename_project(source_project.name, source_project.path)
+      move_relationships_between(source_project, @project)

-        @project
-      end
-    # Projects::DestroyService can raise Exceptions, but we don't want
-    # to pass that kind of exception to the caller. Instead, we change it
-    # for a StandardError exception
-    rescue Exception => e # rubocop:disable Lint/RescueException
-      attempt_restore_repositories(source_project)
+      source_project_rename = rename_project(source_project, tmp_source_name, tmp_source_path)

-      if e.instance_of?(Exception)
-        raise StandardError, e.message
-      else
-        raise
+      if source_project_rename[:status] == :error
+        raise 'Source project rename failed during project overwrite'
      end

+      new_project_rename = rename_project(@project, original_source_name, original_source_path)
+
+      if new_project_rename[:status] == :error
+        rename_project(source_project, original_source_name, original_source_path)
+
+        raise 'New project rename failed during project overwrite'
+      end
+
+      schedule_source_project_deletion(source_project)
+
+      @project
+    rescue StandardError => e
+      move_relationships_between(@project, source_project)
+      remove_source_project_from_fork_network(source_project)
+
+      raise e
    ensure
      track_service(start_time, source_project, e)
    end
@ -48,45 +52,63 @@ module Projects
                                 error: exception.class.name)
    end

-    def move_before_destroy_relationships(source_project)
+    def move_relationships_between(source_project, target_project)
      options = { remove_remaining_elements: false }

-      ::Projects::MoveUsersStarProjectsService.new(@project, @current_user).execute(source_project, **options)
-      ::Projects::MoveAccessService.new(@project, @current_user).execute(source_project, **options)
-      ::Projects::MoveDeployKeysProjectsService.new(@project, @current_user).execute(source_project, **options)
-      ::Projects::MoveNotificationSettingsService.new(@project, @current_user).execute(source_project, **options)
-      ::Projects::MoveForksService.new(@project, @current_user).execute(source_project, **options)
-      ::Projects::MoveLfsObjectsProjectsService.new(@project, @current_user).execute(source_project, **options)
-      add_source_project_to_fork_network(source_project)
+      Project.transaction do
+        ::Projects::MoveUsersStarProjectsService.new(target_project, @current_user).execute(source_project, **options)
+        ::Projects::MoveAccessService.new(target_project, @current_user).execute(source_project, **options)
+        ::Projects::MoveDeployKeysProjectsService.new(target_project, @current_user).execute(source_project, **options)
+        ::Projects::MoveNotificationSettingsService.new(target_project, @current_user).execute(source_project, **options)
+        ::Projects::MoveForksService.new(target_project, @current_user).execute(source_project, **options)
+        ::Projects::MoveLfsObjectsProjectsService.new(target_project, @current_user).execute(source_project, **options)
+
+        add_source_project_to_fork_network(source_project)
+      end
    end

-    def destroy_old_project(source_project)
-      # Delete previous project (synchronously) and unlink relations
-      ::Projects::DestroyService.new(source_project, @current_user).execute
+    def schedule_source_project_deletion(source_project)
+      ::Projects::DestroyService.new(source_project, @current_user).async_execute
    end

-    def rename_project(name, path)
-      # Update de project's name and path to the original name/path
-      ::Projects::UpdateService.new(@project,
-                                    @current_user,
-                                    { name: name, path: path })
-                               .execute
-    end
-
-    def attempt_restore_repositories(project)
-      ::Projects::DestroyRollbackService.new(project, @current_user).execute
+    def rename_project(target_project, name, path)
+      ::Projects::UpdateService.new(target_project, @current_user, { name: name, path: path }).execute
    end

    def add_source_project_to_fork_network(source_project)
-      return unless @project.fork_network
+      return if source_project == @project
+      return unless fork_network

      # Because they have moved all references in the fork network from the source_project
      # we won't be able to query the database (only through its cached data),
      # for its former relationships. That's why we're adding it to the network
      # as a fork of the target project
-      ForkNetworkMember.create!(fork_network: @project.fork_network,
+      ForkNetworkMember.create!(fork_network: fork_network,
                                project: source_project,
                                forked_from_project: @project)
    end
+
+    def remove_source_project_from_fork_network(source_project)
+      return unless fork_network
+
+      fork_member = ForkNetworkMember.find_by( # rubocop: disable CodeReuse/ActiveRecord
+        fork_network: fork_network,
+        project: source_project,
+        forked_from_project: @project)
+
+      fork_member&.destroy
+    end
+
+    def tmp_source_project_name(source_project)
+      random_string = SecureRandom.hex
+      tmp_name = "#{source_project.name}-old-#{random_string}"
+      tmp_path = "#{source_project.path}-old-#{random_string}"
+
+      [tmp_name, tmp_path]
+    end
+
+    def fork_network
+      @project.fork_network_member&.fork_network
+    end
  end
 end
--- a/app/services/service_ping/build_payload_service.rb
+++ b/app/services/service_ping/build_payload_service.rb
@ -19,7 +19,7 @@ module ServicePing
    end

    def raw_payload
-      @raw_payload ||= ::Gitlab::Usage::ServicePingReport.for(mode: :values)
+      @raw_payload ||= ::Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)
    end
  end
 end
--- a/app/services/service_ping/submit_service.rb
+++ b/app/services/service_ping/submit_service.rb
@ -33,7 +33,7 @@ module ServicePing
        }
        submit_payload({ error: error_payload }, url: error_url)

-        usage_data = Gitlab::Usage::ServicePingReport.for(mode: :values)
+        usage_data = Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)
        response = submit_usage_data_payload(usage_data)
      end

--- a/config/feature_flags/development/cache_shared_runners_enabled.yml
+++ b/config/feature_flags/development/cache_shared_runners_enabled.yml
@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/338267
 milestone: '14.2'
 type: development
 group: group::optimize
-default_enabled: false
+default_enabled: true
--- a/db/migrate/20210811193033_add_unique_index_to_vulnerability_finding_links.rb
+++ b/db/migrate/20210811193033_add_unique_index_to_vulnerability_finding_links.rb
@ -1,18 +1,14 @@
 # frozen_string_literal: true

 class AddUniqueIndexToVulnerabilityFindingLinks < Gitlab::Database::Migration[1.0]
-  disable_ddl_transaction!
-
-  NAME_URL_INDEX_NAME = 'finding_link_name_url_idx'
-  URL_INDEX_NAME = 'finding_link_url_idx'
+  # This migration has been moved to db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb
+  # Previously, this was causing an bug where there was a conflict between the table cleanup and the index creation.

  def up
-    add_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :name, :url], unique: true, name: NAME_URL_INDEX_NAME
-    add_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :url], unique: true, where: 'name is null', name: URL_INDEX_NAME
+    # no op
  end

  def down
-    remove_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :name, :url], name: NAME_URL_INDEX_NAME
-    remove_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :url], name: URL_INDEX_NAME
+    # no op
  end
 end
--- a/db/post_migrate/20211104165220_remove_vulnerability_finding_links.rb
+++ b/db/post_migrate/20211104165220_remove_vulnerability_finding_links.rb
@ -1,21 +1,14 @@
 # frozen_string_literal: true

 class RemoveVulnerabilityFindingLinks < Gitlab::Database::Migration[1.0]
-  BATCH_SIZE = 50_000
-  MIGRATION = 'RemoveVulnerabilityFindingLinks'
-
-  disable_ddl_transaction!
+  # This migration has been moved to a TRUNCATE in db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb
+  # Previously, this was causing an bug where there was a conflict between the table cleanup and the index creation.

  def up
-    queue_background_migration_jobs_by_range_at_intervals(
-      define_batchable_model('vulnerability_finding_links'),
-      MIGRATION,
-      2.minutes,
-      batch_size: BATCH_SIZE
-    )
+    # no op
  end

  def down
-    # no ops
+    # no op
  end
 end
--- a/db/post_migrate/20211210173137_remove_vulnerability_finding_links_again.rb
+++ b/db/post_migrate/20211210173137_remove_vulnerability_finding_links_again.rb
@ -1,21 +1,14 @@
 # frozen_string_literal: true

 class RemoveVulnerabilityFindingLinksAgain < Gitlab::Database::Migration[1.0]
-  BATCH_SIZE = 50_000
-  MIGRATION = 'RemoveVulnerabilityFindingLinks'
-
-  disable_ddl_transaction!
+  # This migration has been moved to a TRUNCATE in db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb
+  # Previously, this was causing an bug where there was a conflict between the table cleanup and the index creation.

  def up
-    queue_background_migration_jobs_by_range_at_intervals(
-      define_batchable_model('vulnerability_finding_links'),
-      MIGRATION,
-      2.minutes,
-      batch_size: BATCH_SIZE
-    )
+    # no op
  end

  def down
-    # no ops
+    # no op
  end
 end
--- a/db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb
+++ b/db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb
@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class AddUniqueIndexToVulnerabilityFindingLinksWithTruncate < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  NAME_URL_INDEX_NAME = 'finding_link_name_url_idx'
+  URL_INDEX_NAME = 'finding_link_url_idx'
+
+  def up
+    execute('TRUNCATE TABLE vulnerability_finding_links')
+
+    add_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :name, :url], unique: true, name: NAME_URL_INDEX_NAME
+    add_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :url], unique: true, where: 'name is null', name: URL_INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :name, :url], name: NAME_URL_INDEX_NAME
+    remove_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :url], name: URL_INDEX_NAME
+  end
+end
--- a/db/schema_migrations/20220201193033
+++ b/db/schema_migrations/20220201193033
@ -0,0 +1 @@
+92bbe74c6c3627dd26f709acd2a20f442212eab933f719be815701a3bc429539
--- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md
+++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md
@ -1324,7 +1324,7 @@ has more information about Service Ping.
 ### Generate or get the cached Service Ping

 ```ruby
-Gitlab::Usage::ServicePingReport.for(mode: :values, cached: true)
+Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values, cached: true)
 ```

 ### Generate a fresh new Service Ping
@ -1332,7 +1332,7 @@ Gitlab::Usage::ServicePingReport.for(mode: :values, cached: true)
 This also refreshes the cached Service Ping displayed in the Admin Area

 ```ruby
-Gitlab::Usage::ServicePingReport.for(mode: :values)
+Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)
 ```

 ### Generate and print
--- a/doc/development/service_ping/index.md
+++ b/doc/development/service_ping/index.md
@ -198,8 +198,8 @@ sequenceDiagram
 ## How Service Ping works

 1. The Service Ping [cron job](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/gitlab_service_ping_worker.rb#L24) is set in Sidekiq to run weekly.
-1. When the cron job runs, it calls [`Gitlab::Usage::ServicePingReport.for(mode: :values)`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/service_ping/submit_service.rb).
-1. `Gitlab::Usage::ServicePingReport.for(mode: :values)` [cascades down](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb) to ~400+ other counter method calls.
+1. When the cron job runs, it calls [`Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/service_ping/submit_service.rb).
+1. `Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)` [cascades down](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb) to ~400+ other counter method calls.
 1. The response of all methods calls are [merged together](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L68) into a single JSON payload.
 1. The JSON payload is then [posted to the Versions application](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/service_ping/submit_service.rb#L20)
   If a firewall exception is needed, the required URL depends on several things. If
--- a/doc/integration/elasticsearch.md
+++ b/doc/integration/elasticsearch.md
@ -30,11 +30,14 @@ Elasticsearch and follows Elasticsearch's [End of Life Policy](https://www.elast
 When we change Elasticsearch supported versions in GitLab, we announce them in [deprecation notes](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations) in monthly release posts
 before we remove them.

-NOTE:
-We do not support
-[Amazon's OpenSearch](https://aws.amazon.com/blogs/opensource/opensearch-1-0-launches/)
+### Versions not supported
+
+GitLab does not support:
+
+- [Amazon's OpenSearch](https://aws.amazon.com/blogs/opensource/opensearch-1-0-launches/)
 (a [fork of Elasticsearch](https://www.elastic.co/what-is/opensearch)).
-Follow our progress in [issue #327560](https://gitlab.com/gitlab-org/gitlab/-/issues/327560).
+For updates, see [issue #327560](https://gitlab.com/gitlab-org/gitlab/-/issues/327560).
+- Elasticsearch 8.0. For updates, see [issue #350600](https://gitlab.com/gitlab-org/gitlab/-/issues/350600).

 ## System requirements

--- a/doc/user/admin_area/credentials_inventory.md
+++ b/doc/user/admin_area/credentials_inventory.md
@ -13,9 +13,14 @@ GitLab administrators are responsible for the overall security of their instance
 provides a Credentials inventory to keep track of all the credentials that can be used to access
 their self-managed instance.

-Using Credentials inventory, you can see all the personal access tokens (PAT), SSH keys, and GPG keys
-that exist in your GitLab instance. In addition, you can [revoke](#revoke-a-users-personal-access-token)
-and [delete](#delete-a-users-ssh-key) and see:
+Use Credentials inventory to see for your GitLab instance all:
+
+- Personal access tokens (PAT).
+- Project access tokens (GitLab 14.8 and later).
+- SSH keys.
+- GPG keys.
+
+You can also [revoke](#revoke-a-users-personal-access-token) and [delete](#delete-a-users-ssh-key) and see:

 - Who they belong to.
 - Their access scope.
@ -28,10 +33,6 @@ To access the Credentials inventory:
 1. On the top bar, select **Menu > Admin**.
 1. On the left sidebar, select **Credentials**.

-The following is an example of the Credentials inventory page:
-
-![Credentials inventory page](img/credentials_inventory_v13_10.png)
-
 ## Revoke a user's personal access token

 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214811) in GitLab 13.4.
@ -49,6 +50,15 @@ If you see a **Revoke** button, you can revoke that user's PAT. Whether you see

 When a PAT is revoked from the credentials inventory, the instance notifies the user by email.

+## Revoke a user's project access token
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/243833) in GitLab 14.8.
+
+The **Revoke** button next to a project access token can be selected to revoke that particular project access token. This will both:
+
+- Revoke the token project access token.
+- Enqueue a background worker to delete the project bot user.
+
 ## Delete a user's SSH key

 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225248) in GitLab 13.5.
--- a/doc/user/admin_area/img/credentials_inventory_v13_10.png
+++ b/doc/user/admin_area/img/credentials_inventory_v13_10.png
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@ -117,6 +117,8 @@ module API

    # rubocop: disable CodeReuse/ActiveRecord
    def find_project(id)
+      return unless id
+
      projects = Project.without_deleted

      if id.is_a?(Integer) || id =~ /^\d+$/
--- a/lib/api/usage_data_non_sql_metrics.rb
+++ b/lib/api/usage_data_non_sql_metrics.rb
@ -18,7 +18,7 @@ module API
      get 'non_sql_metrics' do
        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/325534')

-        data = Gitlab::UsageDataNonSqlMetrics.data
+        data = Gitlab::Usage::ServicePingReport.for(output: :non_sql_metrics_values)

        present data
      end
--- a/lib/api/usage_data_queries.rb
+++ b/lib/api/usage_data_queries.rb
@ -18,7 +18,7 @@ module API
      get 'queries' do
        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/325534')

-        queries = Gitlab::UsageDataQueries.data
+        queries = Gitlab::Usage::ServicePingReport.for(output: :metrics_queries)

        present queries
      end
--- a/lib/gitlab/http.rb
+++ b/lib/gitlab/http.rb
@ -25,7 +25,7 @@ module Gitlab
      read_timeout: 20,
      write_timeout: 30
    }.freeze
-    DEFAULT_READ_TOTAL_TIMEOUT = 20.seconds
+    DEFAULT_READ_TOTAL_TIMEOUT = 30.seconds

    include HTTParty # rubocop:disable Gitlab/HTTParty

--- a/lib/gitlab/usage/service_ping_report.rb
+++ b/lib/gitlab/usage/service_ping_report.rb
@ -4,20 +4,32 @@ module Gitlab
  module Usage
    class ServicePingReport
      class << self
-        def for(mode:, cached: false)
-          case mode.to_sym
-          when :values
-            usage_data(cached)
+        def for(output:, cached: false)
+          case output.to_sym
+          when :all_metrics_values
+            all_metrics_values(cached)
+          when :metrics_queries
+            metrics_queries
+          when :non_sql_metrics_values
+            non_sql_metrics_values
          end
        end

        private

-        def usage_data(cached)
+        def all_metrics_values(cached)
          Rails.cache.fetch('usage_data', force: !cached, expires_in: 2.weeks) do
            Gitlab::UsageData.data
          end
        end
+
+        def metrics_queries
+          Gitlab::UsageDataQueries.data
+        end
+
+        def non_sql_metrics_values
+          Gitlab::UsageDataNonSqlMetrics.data
+        end
      end
    end
  end
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@ -203,6 +203,13 @@ module Gitlab
    rescue Addressable::URI::InvalidURIError, TypeError
    end

+    def add_url_parameters(url, params)
+      uri = parse_url(url.to_s)
+      uri.query_values = uri.query_values.to_h.merge(params.to_h.stringify_keys)
+      uri.query_values = nil if uri.query_values.empty?
+      uri.to_s
+    end
+
    def removes_sensitive_data_from_url(uri_string)
      uri = parse_url(uri_string)

--- a/lib/tasks/gitlab/usage_data.rake
+++ b/lib/tasks/gitlab/usage_data.rake
@ -4,17 +4,17 @@ namespace :gitlab do
  namespace :usage_data do
    desc 'GitLab | UsageData | Generate raw SQLs for usage ping in YAML'
    task dump_sql_in_yaml: :environment do
-      puts Gitlab::UsageDataQueries.data.to_yaml
+      puts Gitlab::Usage::ServicePingReport.for(output: :metrics_queries).to_yaml
    end

    desc 'GitLab | UsageData | Generate raw SQLs for usage ping in JSON'
    task dump_sql_in_json: :environment do
-      puts Gitlab::Json.pretty_generate(Gitlab::UsageDataQueries.data)
+      puts Gitlab::Json.pretty_generate(Gitlab::Usage::ServicePingReport.for(output: :metrics_queries))
    end

    desc 'GitLab | UsageData | Generate usage ping in JSON'
    task generate: :environment do
-      puts Gitlab::Json.pretty_generate(Gitlab::Usage::ServicePingReport.for(mode: :values))
+      puts Gitlab::Json.pretty_generate(Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values))
    end

    desc 'GitLab | UsageData | Generate usage ping and send it to Versions Application'
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@ -4785,6 +4785,9 @@ msgstr ""
 msgid "Are you sure you want to revoke this personal access token? This action cannot be undone."
 msgstr ""

+msgid "Are you sure you want to revoke this project access token? This action cannot be undone."
+msgstr ""
+
 msgid "Are you sure you want to stop this environment?"
 msgstr ""

@ -10499,6 +10502,9 @@ msgstr ""
 msgid "Creation date"
 msgstr ""

+msgid "Creator"
+msgstr ""
+
 msgid "Credentials"
 msgstr ""

@ -10511,6 +10517,9 @@ msgstr ""
 msgid "CredentialsInventory|Personal Access Tokens"
 msgstr ""

+msgid "CredentialsInventory|Project Access Tokens"
+msgstr ""
+
 msgid "CredentialsInventory|SSH Keys"
 msgstr ""

@ -11010,10 +11019,16 @@ msgstr ""
 msgid "DastProfiles|AJAX spider"
 msgstr ""

+msgid "DastProfiles|API"
+msgstr ""
+
+msgid "DastProfiles|API endpoint URL"
+msgstr ""
+
 msgid "DastProfiles|Active"
 msgstr ""

-msgid "DastProfiles|Additional request headers (Optional)"
+msgid "DastProfiles|Additional request headers (optional)"
 msgstr ""

 msgid "DastProfiles|Are you sure you want to delete this profile?"
@ -11097,7 +11112,13 @@ msgstr ""
 msgid "DastProfiles|Excluded URLs"
 msgstr ""

-msgid "DastProfiles|Excluded URLs (Optional)"
+msgid "DastProfiles|Excluded URLs (optional)"
+msgstr ""
+
+msgid "DastProfiles|Excluded paths"
+msgstr ""
+
+msgid "DastProfiles|Excluded paths (optional)"
 msgstr ""

 msgid "DastProfiles|Hide debug messages"
@ -11154,9 +11175,6 @@ msgstr ""
 msgid "DastProfiles|Request headers"
 msgstr ""

-msgid "DastProfiles|Rest API"
-msgstr ""
-
 msgid "DastProfiles|Run the AJAX spider, in addition to the traditional spider, to crawl the target site."
 msgstr ""

--- a/package.json
+++ b/package.json
@ -55,7 +55,7 @@
    "@babel/preset-env": "^7.10.1",
    "@gitlab/at.js": "1.5.7",
    "@gitlab/favicon-overlay": "2.0.0",
-    "@gitlab/svgs": "2.3.0",
+    "@gitlab/svgs": "2.5.0",
    "@gitlab/ui": "35.1.0",
    "@gitlab/visual-review-tools": "1.6.1",
    "@rails/actioncable": "6.1.4-1",
--- a/qa/qa/specs/features/browser_ui/3_create/repository/push_to_canary_gitaly_spec.rb
+++ b/qa/qa/specs/features/browser_ui/3_create/repository/push_to_canary_gitaly_spec.rb
@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module QA
+  RSpec.describe 'Create', only: { subdomain: %i[staging staging-canary] } do
+    describe 'Git push to canary Gitaly node over HTTP' do
+      it 'pushes to a project using a canary specific Gitaly repository storage', :smoke, :requires_admin, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/351116' do
+        Flow::Login.sign_in_as_admin
+
+        project = Resource::Project.fabricate_via_api! do |storage_project|
+          storage_project.name = 'canary-specific-repository-storage'
+          storage_project.repository_storage = 'nfs-file-cny01' # TODO: move to ENV var
+        end
+
+        Resource::Repository::Push.fabricate! do |push|
+          push.repository_http_uri = project.repository_http_location.uri
+          push.file_name = 'README.md'
+          push.file_content = "# This is a test project named #{project.name}"
+          push.commit_message = 'Add README.md'
+          push.new_branch = true
+        end
+
+        project.visit!
+
+        Page::Project::Show.perform do |project_page|
+          expect(project_page).to have_file('README.md')
+          expect(project_page).to have_readme_content("This is a test project named #{project.name}")
+        end
+      end
+    end
+  end
+end
--- a/spec/controllers/admin/instance_review_controller_spec.rb
+++ b/spec/controllers/admin/instance_review_controller_spec.rb
@ -23,7 +23,7 @@ RSpec.describe Admin::InstanceReviewController do
        stub_application_setting(usage_ping_enabled: true)
        stub_usage_data_connections
        stub_database_flavor_check
-        ::Gitlab::Usage::ServicePingReport.for(mode: :values)
+        ::Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)
        subject
      end

--- a/spec/lib/api/helpers_spec.rb
+++ b/spec/lib/api/helpers_spec.rb
@ -76,6 +76,12 @@ RSpec.describe API::Helpers do
          expect(subject.find_project(non_existing_id)).to be_nil
        end
      end
+
+      context 'when project id is not provided' do
+        it 'returns nil' do
+          expect(subject.find_project(nil)).to be_nil
+        end
+      end
    end

    context 'when ID is used as an argument' do
--- a/spec/lib/gitlab/usage/service_ping_report_spec.rb
+++ b/spec/lib/gitlab/usage/service_ping_report_spec.rb
@ -5,11 +5,27 @@ require 'spec_helper'
 RSpec.describe Gitlab::Usage::ServicePingReport, :use_clean_rails_memory_store_caching do
  let(:usage_data) { { uuid: "1111" } }

-  context 'for mode: :values' do
+  context 'for output: :all_metrics_values' do
    it 'generates the service ping' do
      expect(Gitlab::UsageData).to receive(:data)

-      described_class.for(mode: :values)
+      described_class.for(output: :all_metrics_values)
+    end
+  end
+
+  context 'for output: :metrics_queries' do
+    it 'generates the service ping' do
+      expect(Gitlab::UsageDataQueries).to receive(:data)
+
+      described_class.for(output: :metrics_queries)
+    end
+  end
+
+  context 'for output: :non_sql_metrics_values' do
+    it 'generates the service ping' do
+      expect(Gitlab::UsageDataNonSqlMetrics).to receive(:data)
+
+      described_class.for(output: :non_sql_metrics_values)
    end
  end

@ -20,8 +36,8 @@ RSpec.describe Gitlab::Usage::ServicePingReport, :use_clean_rails_memory_store_c
      it 'caches the values' do
        allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data)

-        expect(described_class.for(mode: :values)).to eq(usage_data)
-        expect(described_class.for(mode: :values, cached: true)).to eq(usage_data)
+        expect(described_class.for(output: :all_metrics_values)).to eq(usage_data)
+        expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(usage_data)

        expect(Rails.cache.fetch('usage_data')).to eq(usage_data)
      end
@ -29,9 +45,9 @@ RSpec.describe Gitlab::Usage::ServicePingReport, :use_clean_rails_memory_store_c
      it 'writes to cache and returns fresh data' do
        allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data)

-        expect(described_class.for(mode: :values)).to eq(usage_data)
-        expect(described_class.for(mode: :values)).to eq(new_usage_data)
-        expect(described_class.for(mode: :values, cached: true)).to eq(new_usage_data)
+        expect(described_class.for(output: :all_metrics_values)).to eq(usage_data)
+        expect(described_class.for(output: :all_metrics_values)).to eq(new_usage_data)
+        expect(described_class.for(output: :all_metrics_values, cached: true)).to eq(new_usage_data)

        expect(Rails.cache.fetch('usage_data')).to eq(new_usage_data)
      end
@ -43,8 +59,8 @@ RSpec.describe Gitlab::Usage::ServicePingReport, :use_clean_rails_memory_store_c
      it 'returns fresh data' do
        allow(Gitlab::UsageData).to receive(:data).and_return(usage_data, new_usage_data)

-        expect(described_class.for(mode: :values)).to eq(usage_data)
-        expect(described_class.for(mode: :values)).to eq(new_usage_data)
+        expect(described_class.for(output: :all_metrics_values)).to eq(usage_data)
+        expect(described_class.for(output: :all_metrics_values)).to eq(new_usage_data)

        expect(Rails.cache.fetch('usage_data')).to eq(new_usage_data)
      end
--- a/spec/lib/gitlab/utils_spec.rb
+++ b/spec/lib/gitlab/utils_spec.rb
@ -439,6 +439,23 @@ RSpec.describe Gitlab::Utils do
    end
  end

+  describe '.add_url_parameters' do
+    subject { described_class.add_url_parameters(url, params) }
+
+    where(:url, :params, :expected_url) do
+      nil                          | nil                | ''
+      nil                          | { b: 3, a: 2 }     | '?a=2&b=3'
+      'https://gitlab.com'         | nil                | 'https://gitlab.com'
+      'https://gitlab.com'         | { b: 3, a: 2 }     | 'https://gitlab.com?a=2&b=3'
+      'https://gitlab.com?a=1#foo' | { b: 3, 'a': 2 }   | 'https://gitlab.com?a=2&b=3#foo'
+      'https://gitlab.com?a=1#foo' | [[:b, 3], [:a, 2]] | 'https://gitlab.com?a=2&b=3#foo'
+    end
+
+    with_them do
+      it { is_expected.to eq(expected_url) }
+    end
+  end
+
  describe '.removes_sensitive_data_from_url' do
    it 'returns string object' do
      expect(described_class.removes_sensitive_data_from_url('http://gitlab.com')).to be_instance_of(String)
--- a/spec/models/personal_access_token_spec.rb
+++ b/spec/models/personal_access_token_spec.rb
@ -22,6 +22,16 @@ RSpec.describe PersonalAccessToken do
  end

  describe 'scopes' do
+    describe '.project_access_tokens' do
+      let_it_be(:user) { create(:user, :project_bot) }
+      let_it_be(:project_member) { create(:project_member, user: user) }
+      let_it_be(:project_access_token) { create(:personal_access_token, user: user) }
+
+      subject { described_class.project_access_token }
+
+      it { is_expected.to contain_exactly(project_access_token) }
+    end
+
    describe '.for_user' do
      it 'returns personal access tokens of specified user only' do
        user_1 = create(:user)
--- a/spec/services/alert_management/create_alert_issue_service_spec.rb
+++ b/spec/services/alert_management/create_alert_issue_service_spec.rb
@ -43,10 +43,10 @@ RSpec.describe AlertManagement::CreateAlertIssueService do
        expect(execute).to be_success
      end

-      it 'updates alert.issue_id' do
+      it 'sets alert.issue_id in the same ActiveRecord query execution' do
        execute

-        expect(alert.reload.issue_id).to eq(created_issue.id)
+        expect(alert.issue_id).to eq(created_issue.id)
      end

      it 'creates a system note' do
--- a/spec/services/incident_management/incidents/create_service_spec.rb
+++ b/spec/services/incident_management/incidents/create_service_spec.rb
@ -83,6 +83,25 @@ RSpec.describe IncidentManagement::Incidents::CreateService do
      it 'result payload contains an Issue object' do
        expect(create_incident.payload[:issue]).to be_kind_of(Issue)
      end
+
+      context 'with alert' do
+        let(:alert) { create(:alert_management_alert, project: project) }
+
+        subject(:create_incident) { described_class.new(project, user, title: title, description: description, alert: alert).execute }
+
+        it 'associates the alert with the incident' do
+          expect(create_incident[:issue].alert_management_alert).to eq(alert)
+        end
+
+        context 'the alert prevents the issue from saving' do
+          let(:alert) { create(:alert_management_alert, :with_validation_errors, project: project) }
+
+          it 'responds with errors' do
+            expect(create_incident).to be_error
+            expect(create_incident.message).to eq('Hosts hosts array is over 255 chars')
+          end
+        end
+      end
    end
  end
 end
--- a/spec/services/projects/overwrite_project_service_spec.rb
+++ b/spec/services/projects/overwrite_project_service_spec.rb
@ -81,16 +81,58 @@ RSpec.describe Projects::OverwriteProjectService do
        end
      end

-      it 'removes the original project' do
-        subject.execute(project_from)
+      it 'schedules original project for deletion' do
+        expect_next_instance_of(Projects::DestroyService) do |service|
+          expect(service).to receive(:async_execute)
+        end

-        expect { Project.find(project_from.id) }.to raise_error(ActiveRecord::RecordNotFound)
+        subject.execute(project_from)
      end

      it 'renames the project' do
+        original_path = project_from.full_path
+
        subject.execute(project_from)

-        expect(project_to.full_path).to eq project_from.full_path
+        expect(project_to.full_path).to eq(original_path)
+      end
+
+      it 'renames source project to temp name' do
+        allow(SecureRandom).to receive(:hex).and_return('test')
+
+        subject.execute(project_from)
+
+        expect(project_from.full_path).to include('-old-test')
+      end
+
+      context 'when project rename fails' do
+        before do
+          expect(subject).to receive(:move_relationships_between).with(project_from, project_to)
+          expect(subject).to receive(:move_relationships_between).with(project_to, project_from)
+        end
+
+        context 'source rename' do
+          it 'moves relations back to source project and raises an exception' do
+            allow(subject).to receive(:rename_project).and_return(status: :error)
+
+            expect { subject.execute(project_from) }.to raise_error(StandardError, 'Source project rename failed during project overwrite')
+          end
+        end
+
+        context 'new project rename' do
+          it 'moves relations back, renames source project back to original name and raises' do
+            name = project_from.name
+            path = project_from.path
+
+            allow(subject).to receive(:rename_project).and_call_original
+            allow(subject).to receive(:rename_project).with(project_to, name, path).and_return(status: :error)
+
+            expect { subject.execute(project_from) }.to raise_error(StandardError, 'New project rename failed during project overwrite')
+
+            expect(project_from.name).to eq(name)
+            expect(project_from.path).to eq(path)
+          end
+        end
      end
    end

@ -121,7 +163,7 @@ RSpec.describe Projects::OverwriteProjectService do
      end
    end

-    context 'forks' do
+    context 'forks', :sidekiq_inline do
      context 'when moving a root forked project' do
        it 'moves the descendant forks' do
          expect(project_from.forks.count).to eq 2
@ -147,6 +189,7 @@ RSpec.describe Projects::OverwriteProjectService do
          expect(project_to.fork_network.fork_network_members.map(&:project)).not_to include project_from
        end
      end
+
      context 'when moving a intermediate forked project' do
        let(:project_to) { create(:project, namespace: lvl1_forked_project_1.namespace) }

@ -180,22 +223,26 @@ RSpec.describe Projects::OverwriteProjectService do
    end

    context 'if an exception is raised' do
+      before do
+        allow(subject).to receive(:rename_project).and_raise(StandardError)
+      end
+
      it 'rollbacks changes' do
        updated_at = project_from.updated_at

-        allow(subject).to receive(:rename_project).and_raise(StandardError)
-
        expect { subject.execute(project_from) }.to raise_error(StandardError)
        expect(Project.find(project_from.id)).not_to be_nil
        expect(project_from.reload.updated_at.change(usec: 0)).to eq updated_at.change(usec: 0)
      end

-      it 'tries to restore the original project repositories' do
-        allow(subject).to receive(:rename_project).and_raise(StandardError)
-
-        expect(subject).to receive(:attempt_restore_repositories).with(project_from)
+      it 'removes fork network member' do
+        expect(ForkNetworkMember).to receive(:create!)
+        expect(ForkNetworkMember).to receive(:find_by)
+        expect(subject).to receive(:remove_source_project_from_fork_network).and_call_original

        expect { subject.execute(project_from) }.to raise_error(StandardError)
+
+        expect(project_from.fork_network_member).to be_nil
      end
    end
  end
--- a/spec/services/service_ping/submit_service_ping_service_spec.rb
+++ b/spec/services/service_ping/submit_service_ping_service_spec.rb
@ -118,7 +118,7 @@ RSpec.describe ServicePing::SubmitService do
    it 'generates service ping' do
      stub_response(body: with_dev_ops_score_params)

-      expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_call_original
+      expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_call_original

      subject.execute
    end
@ -151,7 +151,7 @@ RSpec.describe ServicePing::SubmitService do
    it 'forces a refresh of usage data statistics before submitting' do
      stub_response(body: with_dev_ops_score_params)

-      expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_call_original
+      expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_call_original

      subject.execute
    end
@ -167,7 +167,7 @@ RSpec.describe ServicePing::SubmitService do
        recorded_at = Time.current
        usage_data = { uuid: 'uuid', recorded_at: recorded_at }

-        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_return(usage_data)
+        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_return(usage_data)

        subject.execute

@ -190,7 +190,7 @@ RSpec.describe ServicePing::SubmitService do
        recorded_at = Time.current
        usage_data = { uuid: 'uuid', recorded_at: recorded_at }

-        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_return(usage_data)
+        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_return(usage_data)

        subject.execute

@ -235,7 +235,7 @@ RSpec.describe ServicePing::SubmitService do
        recorded_at = Time.current
        usage_data = { uuid: 'uuid', recorded_at: recorded_at }

-        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_return(usage_data)
+        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_return(usage_data)

        subject.execute

@ -260,7 +260,7 @@ RSpec.describe ServicePing::SubmitService do

    context 'and usage data is empty string' do
      before do
-        allow(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_return({})
+        allow(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_return({})
      end

      it_behaves_like 'does not send a blank usage ping payload'
@ -269,7 +269,7 @@ RSpec.describe ServicePing::SubmitService do
    context 'and usage data is nil' do
      before do
        allow(ServicePing::BuildPayloadService).to receive(:execute).and_return(nil)
-        allow(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_return(nil)
+        allow(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_return(nil)
      end

      it_behaves_like 'does not send a blank usage ping payload'
@ -285,7 +285,7 @@ RSpec.describe ServicePing::SubmitService do
      it 'calls Gitlab::Usage::ServicePingReport .for method' do
        usage_data = build_usage_data

-        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_return(usage_data)
+        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_return(usage_data)

        subject.execute
      end
@ -329,7 +329,7 @@ RSpec.describe ServicePing::SubmitService do
      it 'calls Gitlab::Usage::ServicePingReport .for method' do
        usage_data = build_usage_data

-        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(mode: :values).and_return(usage_data)
+        expect(Gitlab::Usage::ServicePingReport).to receive(:for).with(output: :all_metrics_values).and_return(usage_data)

        # SubmissionError is raised as a result of 404 in response from HTTP Request
        expect { subject.execute }.to raise_error(described_class::SubmissionError)
--- a/yarn.lock
+++ b/yarn.lock
@ -957,10 +957,10 @@
    stylelint-declaration-strict-value "1.8.0"
    stylelint-scss "4.1.0"

-"@gitlab/svgs@2.3.0":
-  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-2.3.0.tgz#2ddb38c1e27a5f1945863c3093107117c0175be4"
-  integrity sha512-VXryDplnM+sImEleyxDnW4oyDvIwUhSCZ/+hxYmXesysQiK+vm+hEfdc2N+AnlD2xbdbnuMQnegckOL38Ic2/g==
+"@gitlab/svgs@2.5.0":
+  version "2.5.0"
+  resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-2.5.0.tgz#e0569916fa858462b1801cc90ef8dd9706a12e96"
+  integrity sha512-cH/EBs//wdkH6kG+kDpvRCIl63/A8JgjAhBJ+ZWucPgtNCDD6x6RDMGdQrxSqhYwcCKDoLStfcxmblBkuiSRXQ==

 "@gitlab/ui@35.1.0":
  version "35.1.0"
				`@ -0,0 +1 @@`
				`92bbe74c6c3627dd26f709acd2a20f442212eab933f719be815701a3bc429539`