Add latest changes from gitlab-org/gitlab@master

2023-12-16 06:12:45 +00:00 · 2023-12-16 06:12:45 +00:00 · 3e4c70d070
parent 555976016b
commit 3e4c70d070
39 changed files with 350 additions and 67 deletions
--- a/app/assets/javascripts/boards/components/board_content.vue
+++ b/app/assets/javascripts/boards/components/board_content.vue
@ -5,7 +5,7 @@ import produce from 'immer';
 import Draggable from 'vuedraggable';
 import BoardAddNewColumn from 'ee_else_ce/boards/components/board_add_new_column.vue';
 import { s__ } from '~/locale';
-import { defaultSortableOptions } from '~/sortable/constants';
+import { defaultSortableOptions, DRAG_DELAY } from '~/sortable/constants';
 import {
  DraggableItemTypes,
  flashAnimationDuration,
@ -89,7 +89,7 @@ export default {
        group: 'boards-list',
        tag: 'div',
        value: this.boardListsToUse,
-        delay: 100,
+        delay: DRAG_DELAY,
        delayOnTouchOnly: true,
        filter: 'input',
        preventOnFilter: false,
--- a/app/assets/javascripts/boards/components/board_list.vue
+++ b/app/assets/javascripts/boards/components/board_list.vue
@ -3,7 +3,7 @@ import { GlLoadingIcon, GlIntersectionObserver } from '@gitlab/ui';
 import Draggable from 'vuedraggable';
 import { STATUS_CLOSED } from '~/issues/constants';
 import { sprintf, __, s__ } from '~/locale';
-import { defaultSortableOptions } from '~/sortable/constants';
+import { defaultSortableOptions, DRAG_DELAY } from '~/sortable/constants';
 import { sortableStart, sortableEnd } from '~/sortable/utils';
 import Tracking from '~/tracking';
 import listQuery from 'ee_else_ce/boards/graphql/board_lists_deferred.query.graphql';
@ -218,7 +218,7 @@ export default {
        'ghost-class': 'board-card-drag-active',
        'data-list-id': this.list.id,
        value: this.boardListItems,
-        delay: 100,
+        delay: DRAG_DELAY,
        delayOnTouchOnly: true,
      };

--- a/app/assets/javascripts/sortable/constants.js
+++ b/app/assets/javascripts/sortable/constants.js
@ -1,4 +1,5 @@
 export const DRAG_CLASS = 'is-dragging';
+export const DRAG_DELAY = 100;

 /**
 * Default config options for sortablejs.
--- a/app/assets/javascripts/vue_shared/issuable/list/components/issuable_list_root.vue
+++ b/app/assets/javascripts/vue_shared/issuable/list/components/issuable_list_root.vue
@ -6,6 +6,7 @@ import PageSizeSelector from '~/vue_shared/components/page_size_selector.vue';
 import { getIdFromGraphQLId } from '~/graphql_shared/utils';
 import { updateHistory, setUrlParams } from '~/lib/utils/url_utility';
 import { __ } from '~/locale';
+import { DRAG_DELAY } from '~/sortable/constants';
 import FilteredSearchBar from '~/vue_shared/components/filtered_search_bar/filtered_search_bar_root.vue';
 import glFeatureFlagMixin from '~/vue_shared/mixins/gl_feature_flags_mixin';

@ -24,6 +25,8 @@ export default {
    forceFallback: true,
    ghostClass: 'gl-visibility-hidden',
    tag: 'ul',
+    delay: DRAG_DELAY,
+    delayOnTouchOnly: true,
  },
  components: {
    GlAlert,
--- a/app/assets/javascripts/work_items/components/work_item_links/work_item_children_wrapper.vue
+++ b/app/assets/javascripts/work_items/components/work_item_links/work_item_children_wrapper.vue
@ -6,7 +6,7 @@ import * as Sentry from '~/sentry/sentry_browser_wrapper';
 import { isLoggedIn } from '~/lib/utils/common_utils';
 import { DEFAULT_DEBOUNCE_AND_THROTTLE_MS } from '~/lib/utils/constants';
 import { s__ } from '~/locale';
-import { defaultSortableOptions } from '~/sortable/constants';
+import { defaultSortableOptions, DRAG_DELAY } from '~/sortable/constants';

 import { WORK_ITEM_TYPE_VALUE_OBJECTIVE } from '../../constants';
 import { findHierarchyWidgets } from '../../utils';
@ -77,6 +77,8 @@ export default {
        'ghost-class': 'tree-item-drag-active',
        'data-parent-id': this.workItemId,
        value: this.children,
+        delay: DRAG_DELAY,
+        delayOnTouchOnly: true,
      };

      return this.canReorder ? options : {};
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@ -824,6 +824,13 @@ module Ci
      add_message(:warning, content)
    end

+    # Like #drop!, but does not persist the pipeline nor trigger any state
+    # machine callbacks.
+    def set_failed(drop_reason)
+      self.failure_reason = drop_reason.to_s
+      self.status = 'failed'
+    end
+
    # We can't use `messages.error` scope here because messages should also be
    # read when the pipeline is not persisted. Using the scope will return no
    # results as it would query persisted data.
@ -1073,6 +1080,10 @@ module Ci
      persisted? && failure_reason.blank?
    end

+    def filtered_as_empty?
+      filtered_by_rules? || filtered_by_workflow_rules?
+    end
+
    def detailed_status(current_user)
      Gitlab::Ci::Status::Pipeline::Factory
        .new(self.present, current_user)
--- a/app/models/concerns/enums/ci/pipeline.rb
+++ b/app/models/concerns/enums/ci/pipeline.rb
@ -15,10 +15,20 @@ module Enums
          job_activity_limit_exceeded: 22,
          deployments_limit_exceeded: 23,
          # 24 was previously used by the deprecated `user_blocked`
-          project_deleted: 25
+          project_deleted: 25,
+          filtered_by_rules: 26,
+          filtered_by_workflow_rules: 27
        }
      end

+      def self.persistable_failure_reasons
+        failure_reasons.except(:filtered_by_rules, :filtered_by_workflow_rules)
+      end
+
+      def self.persistable_failure_reason?(reason)
+        persistable_failure_reasons.include?(reason)
+      end
+
      # Returns the `Hash` to use for creating the `sources` enum for
      # `Ci::Pipeline`.
      def self.sources
--- a/app/models/service_desk/custom_email_credential.rb
+++ b/app/models/service_desk/custom_email_credential.rb
@ -66,7 +66,7 @@ module ServiceDesk
        ascii_only: true,
        enforce_sanitization: true,
        allow_localhost: false,
-        allow_local_network: false
+        allow_local_network: !::Gitlab.com? # rubocop:disable Gitlab/AvoidGitlabInstanceChecks -- self-managed may also use local network
      )
    rescue Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError => e
      errors.add(:smtp_address, e)
--- a/app/presenters/ci/pipeline_presenter.rb
+++ b/app/presenters/ci/pipeline_presenter.rb
@ -16,7 +16,10 @@ module Ci
        size_limit_exceeded: 'The pipeline size limit was exceeded.',
        job_activity_limit_exceeded: 'The pipeline job activity limit was exceeded.',
        deployments_limit_exceeded: 'The pipeline deployments limit was exceeded.',
-        project_deleted: 'The project associated with this pipeline was deleted.' }
+        project_deleted: 'The project associated with this pipeline was deleted.',
+        filtered_by_rules: 'Pipeline will not run for the selected trigger. ' \
+                           'The rules configuration prevented any jobs from being added to the pipeline.',
+        filtered_by_workflow_rules: 'Pipeline filtered out by workflow rules.' }
    end

    presents ::Ci::Pipeline, as: :pipeline
--- a/config/feature_flags/development/always_set_pipeline_failure_reason.yml
+++ b/config/feature_flags/development/always_set_pipeline_failure_reason.yml
@ -0,0 +1,8 @@
+---
+name: always_set_pipeline_failure_reason
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/138390
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/434228
+milestone: '16.7'
+type: development
+group: group::pipeline execution
+default_enabled: false
--- a/config/initializers/7_redis.rb
+++ b/config/initializers/7_redis.rb
@ -39,5 +39,5 @@ end
 # 2. Rails.cache
 # 3. HTTP clients
 Gitlab::Redis::ALL_CLASSES.each do |redis_instance|
-  redis_instance.with { nil }
+  redis_instance.with { nil } unless redis_instance == Gitlab::Redis::ClusterSharedState
 end
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@ -171,6 +171,8 @@
  - 1
 - - ci_parse_secure_file_metadata
  - 1
+- - ci_runners_export_usage_csv
+  - 1
 - - ci_runners_process_runner_version_update
  - 1
 - - ci_unlock_pipelines_in_queue
--- a/doc/api/graphql/reference/index.md
+++ b/doc/api/graphql/reference/index.md
@ -6731,6 +6731,28 @@ Input type: `RunnerUpdateInput`
 | <a id="mutationrunnerupdateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. |
 | <a id="mutationrunnerupdaterunner"></a>`runner` | [`CiRunner`](#cirunner) | Runner after mutation. |

+### `Mutation.runnersExportUsage`
+
+WARNING:
+**Introduced** in 16.7.
+This feature is an Experiment. It can be changed or removed at any time.
+
+Input type: `RunnersExportUsageInput`
+
+#### Arguments
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| <a id="mutationrunnersexportusageclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. |
+| <a id="mutationrunnersexportusagetype"></a>`type` | [`CiRunnerType`](#cirunnertype) | Scope of the runners to include in the report. |
+
+#### Fields
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| <a id="mutationrunnersexportusageclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. |
+| <a id="mutationrunnersexportusageerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. |
+
 ### `Mutation.runnersRegistrationTokenReset`

 Input type: `RunnersRegistrationTokenResetInput`
--- a/doc/ci/secure_files/index.md
+++ b/doc/ci/secure_files/index.md
@ -60,3 +60,26 @@ WARNING:
 The content of files loaded with the `download-secure-files` tool are not [masked](../variables/index.md#mask-a-cicd-variable)
 in the job log output. Make sure to avoid outputting secure file contents in the job log,
 especially when logging output that could contain sensitive information.
+
+## Security details
+
+Project-level Secure Files are encrypted on upload using the [Lockbox](https://github.com/ankane/lockbox)
+Ruby gem by using the [`Ci::SecureFileUploader`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/uploaders/ci/secure_file_uploader.rb)
+interface. This interface generates a SHA256 checksum of the source file during upload
+that is persisted with the record in the database so it can be used to verify the contents
+of the file when downloaded.
+
+A [unique encryption key](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/ci/secure_file.rb#L27)
+is generated for each file when it is created and persisted in the database. The encrypted uploaded files
+are stored in either local storage or object storage depending on the [GitLab instance configuration](../../administration/secure_files.md).
+
+Individual files can be retrieved with the [secure files download API](../../api/secure_files.md#download-secure-file).
+Metadata can be retrieved with the [list](../../api/secure_files.md#list-project-secure-files)
+or [show](../../api/secure_files.md#show-secure-file-details) API endpoints. Files can also be retrieved
+with the [`download-secure-files`](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files)
+tool. This tool automatically verifies the checksum of each file as it is downloaded.
+
+Any project member with at least the Developer role can access Project-level secure files.
+Interactions with Project-level secure files are not included in Audit Events, but
+[issue 117](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/readme/-/issues/117).
+proposes adding this functionality.
--- a/doc/user/group/manage.md
+++ b/doc/user/group/manage.md
@ -443,7 +443,7 @@ Approval settings should not be confused with [approval rules](../project/merge_
 for the ability to set merge request approval rules for groups is tracked in
 [epic 4367](https://gitlab.com/groups/gitlab-org/-/epics/4367).

-## Enable Code Suggestions **(FREE SAAS)**
+## Enable Code Suggestions for a group **(FREE SAAS)**

 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/405126) in GitLab 15.11.
 > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/408158) from GitLab Ultimate to GitLab Premium in 16.0.
@ -454,12 +454,9 @@ This feature is in [Beta](../../policy/experiment-beta-support.md#beta).
 Beta users should read about the [known limitations](../project/repository/code_suggestions/index.md#known-limitations).
 We look forward to hearing your [feedback](../project/repository/code_suggestions/index.md#feedback).

-You can give all users in a group and its subgroups access to [Code Suggestions](../project/repository/code_suggestions/index.md).
-
- This setting
-  [cascades to all projects](../project/merge_requests/approvals/settings.md#settings-cascading) in the group.
- Each user can
-  [enable Code Suggestions](../../user/profile/preferences.md#enable-code-suggestions).
+You can give all users in a group and its subgroups access to
+[Code Suggestions](../project/repository/code_suggestions/index.md). This setting
+[cascades to all projects](../project/merge_requests/approvals/settings.md#settings-cascading) in the group.

 Code Suggestions are enabled by default at the group level.

@ -471,6 +468,9 @@ To update this setting:
 1. Under **Code Suggestions**, select the **Projects in this group can use Code Suggestions** checkbox.
 1. Select **Save changes**.

+Individual users can disable Code Suggestions by disabling the feature in their
+[installed IDE editor extension](../project/repository/code_suggestions/index.md#supported-editor-extensions).
+
 ## Enable Experiment and Beta features **(ULTIMATE SAAS)**

 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118222) in GitLab 16.0.
--- a/doc/user/profile/preferences.md
+++ b/doc/user/profile/preferences.md
@ -318,13 +318,15 @@ To access your **Followers** and **Following** tabs:
 - On the left sidebar, select your avatar > select your name or username.
 - Select **Followers** or **Following**.

-## Enable Code Suggestions
+## Enable Code Suggestions **(FREE SAAS)**

-> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121079) in GitLab 16.1 as [Beta](../../policy/experiment-beta-support.md#beta).
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121079) in GitLab 16.1 as [Beta](../../policy/experiment-beta-support.md#beta).
+> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139916) in GitLab 16.8. Available to a percentage of users.

-Code Suggestions are disabled by default at the user account level.
+A percentage of users can use Code Suggestions without any additional configuration.

-To update this setting:
+If the following options are available to you, it means you are **not** part of the percentage of users
+and you must manually enable Code Suggestions for your account:

 1. On the left sidebar, select your avatar.
 1. Select **Preferences**.
@ -332,7 +334,7 @@ To update this setting:
 1. Select **Save changes**.

 NOTE:
-If Code Suggestions are disabled [for any groups that you belong to](../../user/group/manage.md#enable-code-suggestions), then you cannot enable them for yourself. (Your setting has no effect.)
+If Code Suggestions are disabled [for any groups that you belong to](../../user/group/manage.md#enable-code-suggestions-for-a-group), then you cannot enable them for yourself. (Your setting has no effect.)

 ## Integrate your GitLab instance with third-party services

--- a/doc/user/project/repository/code_suggestions/saas.md
+++ b/doc/user/project/repository/code_suggestions/saas.md
@ -19,12 +19,11 @@ Learn about [data usage when using Code Suggestions](index.md#code-suggestions-d

 ## Enable Code Suggestions

-> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121079) in GitLab 16.1 as [Beta](../../../../policy/experiment-beta-support.md#beta).
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121079) in GitLab 16.1 as [Beta](../../../../policy/experiment-beta-support.md#beta).
+> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139916) in GitLab 16.8. Available to a percentage of users.

-You must enable Code Suggestions for both your user account and your top-level group:
-
- [Enable Code Suggestions for your top-level group](../../../group/manage.md#enable-code-suggestions) (you must be a group owner).
- [Enable Code Suggestions for your own account](../../../profile/preferences.md#enable-code-suggestions).
+A group owner must
+[enable Code Suggestions for your top-level group](../../../group/manage.md#enable-code-suggestions-for-a-group).

 NOTE:
 If you are having issues enabling Code Suggestions, view the
@ -34,13 +33,18 @@ If you are having issues enabling Code Suggestions, view the

 Prerequisites:

- You must have a [supported IDE editor extension](index.md#supported-editor-extensions).
+- You must have configured Code Suggestions in a
+  [supported IDE editor extension](index.md#supported-editor-extensions).
 - Code Suggestions must be enabled for:
-  - [The top-level group](../../../group/manage.md#enable-code-suggestions).
-  - [Your own account](../../../profile/preferences.md#enable-code-suggestions).
+  - [The top-level group](../../../group/manage.md#enable-code-suggestions-for-a-group).
+  - [Your own account](../../../profile/preferences.md#enable-code-suggestions), if your
+    account is not part of the percentage rollout.

 To use Code Suggestions:

+1. Determine if your user account is part of the percentage rollout. See
+   [Enable Code Suggestions](../../../profile/preferences.md#enable-code-suggestions)
+   for more information.
 1. Author your code. As you type, suggestions are displayed.
   Code Suggestions provide code snippets or complete the current line, depending on the cursor position.
 1. Describe the requirements in natural language. Be concise and specific. Code Suggestions generates functions and code snippets as appropriate.
@ -52,3 +56,8 @@ Things to remember:

 - AI is non-deterministic, so you may not get the same suggestion every time with the same input.
 - Just like product requirements, writing clear, descriptive, and specific tasks results in quality generated code.
+
+## Disable Code Suggestions
+
+Individual users can disable Code Suggestions by disabling the feature in their
+[installed IDE editor extension](index.md#supported-editor-extensions).
--- a/doc/user/project/repository/code_suggestions/self_managed.md
+++ b/doc/user/project/repository/code_suggestions/self_managed.md
@ -27,14 +27,15 @@ Learn about [data usage when using Code Suggestions](index.md#code-suggestions-d

 ## Enable Code Suggestions on self-managed GitLab

-> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/10653) in GitLab 16.1 as [Beta](../../../../policy/experiment-beta-support.md#beta).
+> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/10653) in GitLab 16.1 as [Beta](../../../../policy/experiment-beta-support.md#beta).
+> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139916) in GitLab 16.8. Available to a percentage of users.

 When you enable Code Suggestions for your self-managed instance, you:

 - Agree to the [GitLab testing agreement](https://about.gitlab.com/handbook/legal/testing-agreement/).
 - Acknowledge that GitLab sends data from the instance, including personal data, to GitLab.com infrastructure.

-How you enable Code Suggestions differs depending on your version of GitLab.
+How you enable Code Suggestions for your instance differs depending on your version of GitLab.

 ### GitLab 16.3 and later **(PREMIUM)**

@ -180,3 +181,8 @@ The Code Suggestions service then securely returns an AI-generated code suggesti

 Neither GitLab nor Google Vertex AI Codey APIs have any visibility into a self-managed customer's code other than
 what is sent to generate the code suggestion.
+
+## Disable Code Suggestions
+
+Individual users can disable Code Suggestions by disabling the feature in their
+[installed IDE editor extension](index.md#supported-editor-extensions).
--- a/doc/user/project/repository/code_suggestions/troubleshooting.md
+++ b/doc/user/project/repository/code_suggestions/troubleshooting.md
@ -10,12 +10,12 @@ When working with GitLab Duo Code Suggestions, you might encounter the following

 ## Code Suggestions aren't displayed

-If Code Suggestions are not displayed, try the following troubleshooting steps.
+If Code Suggestions are not displayed, and you have [installed a supported IDE extension](index.md#supported-editor-extensions), try the following troubleshooting steps.

 In GitLab, ensure Code Suggestions is enabled:

 - [For your user account](../../../profile/preferences.md#enable-code-suggestions).
- [For *all* top-level groups your account belongs to](../../../group/manage.md#enable-code-suggestions). If you don't have a role that lets you view the top-level group's settings, contact a group owner.
+- [For **all** top-level groups your account belongs to](../../../group/manage.md#enable-code-suggestions-for-a-group). If you don't have a role that lets you view the top-level group's settings, contact a group owner.

 ### Code Suggestions not displayed in VS Code or GitLab WebIDE

--- a/doc/user/project/service_desk/configure.md
+++ b/doc/user/project/service_desk/configure.md
@ -184,6 +184,7 @@ To enable this setting:
 > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/387003) in GitLab 16.4.
 > - Ability to select the SMTP authentication method [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/429680) in GitLab 16.6.
 > - [Feature flag `service_desk_custom_email` removed](https://gitlab.com/gitlab-org/gitlab/-/issues/387003) in GitLab 16.7.
+> - Local network allowed for SMTP host on GitLab self-managed [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/435206) in GitLab 16.7

 Configure a custom email address to show as the sender of your support communication.
 Maintain brand identity and instill confidence among support requesters with a domain they recognize.
@ -213,6 +214,8 @@ The custom email address you want to use must meet all of the following requirem
 - You have SMTP credentials (ideally, you should use an app password).
  The username and password are stored in the database using the Advanced Encryption Standard (AES)
  with a 256-bit key.
+- The **SMTP host** must be resolvable from the network of your GitLab instance (on GitLab self-managed)
+  or the public internet (on GitLab SaaS).
 - You must have at least the Maintainer role for the project.
 - Service Desk must be configured for the project.

@ -226,8 +229,7 @@ Configure and verify a custom email address when you want to send Service Desk e
 1. Note the presented Service Desk address of this project, and with your email provider
   (for example, Gmail), set up email forwarding from the custom email address to the
   Service Desk address.
-1. Back in GitLab, complete the fields. **SMTP host** must be resolvable from the network of your GitLab instance (on GitLab self-managed)
-   or the public internet (on GitLab SaaS).
+1. Back in GitLab, complete the fields.
 1. Select **Save & test settings**.

 The configuration has been saved and the verification of the custom email address is triggered.
--- a/lib/click_house/migration_support/exclusive_lock.rb
+++ b/lib/click_house/migration_support/exclusive_lock.rb
@ -17,7 +17,7 @@ module ClickHouse
        def register_running_worker(worker_class, worker_id)
          ttl = worker_class.click_house_worker_attrs[:migration_lock_ttl].from_now.utc

-          Gitlab::Redis::ClusterSharedState.with do |redis|
+          Gitlab::Redis::SharedState.with do |redis|
            redis.zadd(ACTIVE_WORKERS_REDIS_KEY, ttl.to_i, worker_id, gt: true)

            yield
@ -41,7 +41,7 @@ module ClickHouse
        end

        def active_sidekiq_workers?
-          Gitlab::Redis::ClusterSharedState.with do |redis|
+          Gitlab::Redis::SharedState.with do |redis|
            min = Time.now.utc.to_i

            # expire keys in the past
--- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb
+++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb
@ -11,7 +11,16 @@ module Gitlab
          def perform!
            @command.workflow_rules_result = workflow_rules_result

-            error('Pipeline filtered out by workflow rules.') unless workflow_passed?
+            return if workflow_passed?
+
+            if Feature.enabled?(:always_set_pipeline_failure_reason, @command.project)
+              drop_reason = :filtered_by_workflow_rules
+            end
+
+            error(
+              'Pipeline filtered out by workflow rules.',
+              drop_reason: drop_reason
+            )
          end

          def break?
--- a/lib/gitlab/ci/pipeline/chain/helpers.rb
+++ b/lib/gitlab/ci/pipeline/chain/helpers.rb
@ -35,7 +35,7 @@ module Gitlab
          def drop_pipeline!(drop_reason)
            return if pipeline.readonly?

-            if drop_reason && command.save_incompleted
+            if Enums::Ci::Pipeline.persistable_failure_reason?(drop_reason) && command.save_incompleted
              # Project iid must be called outside a transaction, so we ensure it is set here
              # otherwise it may be set within the state transition transaction of the drop! call
              # which it will lock the InternalId row for the whole transaction
@ -44,6 +44,8 @@ module Gitlab
              pipeline.drop!(drop_reason)
            else
              command.increment_pipeline_failure_reason_counter(drop_reason)
+
+              pipeline.set_failed(drop_reason) if Feature.enabled?(:always_set_pipeline_failure_reason, command.project)
            end
          end
        end
--- a/lib/gitlab/ci/pipeline/chain/populate.rb
+++ b/lib/gitlab/ci/pipeline/chain/populate.rb
@ -18,8 +18,15 @@ module Gitlab
            pipeline.stages = @command.pipeline_seed.stages

            if stage_names.empty?
-              return error('Pipeline will not run for the selected trigger. ' \
-                'The rules configuration prevented any jobs from being added to the pipeline.')
+              if Feature.enabled?(:always_set_pipeline_failure_reason, @command.project)
+                drop_reason = :filtered_by_rules
+              end
+
+              return error(
+                'Pipeline will not run for the selected trigger. ' \
+                  'The rules configuration prevented any jobs from being added to the pipeline.',
+                drop_reason: drop_reason
+              )
            end

            if pipeline.invalid?
--- a/lib/gitlab/exclusive_lease.rb
+++ b/lib/gitlab/exclusive_lease.rb
@ -31,7 +31,7 @@ module Gitlab
    EOS

    def self.get_uuid(key)
-      Gitlab::Redis::ClusterSharedState.with do |redis|
+      Gitlab::Redis::SharedState.with do |redis|
        redis.get(redis_shared_state_key(key)) || false
      end
    end
@ -61,7 +61,7 @@ module Gitlab
    def self.cancel(key, uuid)
      return unless key.present?

-      Gitlab::Redis::ClusterSharedState.with do |redis|
+      Gitlab::Redis::SharedState.with do |redis|
        redis.eval(LUA_CANCEL_SCRIPT, keys: [ensure_prefixed_key(key)], argv: [uuid])
      end
    end
@ -79,7 +79,7 @@ module Gitlab
    # Removes any existing exclusive_lease from redis
    # Don't run this in a live system without making sure no one is using the leases
    def self.reset_all!(scope = '*')
-      Gitlab::Redis::ClusterSharedState.with do |redis|
+      Gitlab::Redis::SharedState.with do |redis|
        redis.scan_each(match: redis_shared_state_key(scope)).each do |key|
          redis.del(key)
        end
@ -96,7 +96,7 @@ module Gitlab
    # false if the lease is already taken.
    def try_obtain
      # Performing a single SET is atomic
-      Gitlab::Redis::ClusterSharedState.with do |redis|
+      Gitlab::Redis::SharedState.with do |redis|
        redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid
      end
    end
@ -109,7 +109,7 @@ module Gitlab
    # Try to renew an existing lease. Return lease UUID on success,
    # false if the lease is taken by a different UUID or inexistent.
    def renew
-      Gitlab::Redis::ClusterSharedState.with do |redis|
+      Gitlab::Redis::SharedState.with do |redis|
        result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout])
        result == @uuid
      end
@ -117,7 +117,7 @@ module Gitlab

    # Returns true if the key for this lease is set.
    def exists?
-      Gitlab::Redis::ClusterSharedState.with do |redis|
+      Gitlab::Redis::SharedState.with do |redis|
        redis.exists?(@redis_shared_state_key) # rubocop:disable CodeReuse/ActiveRecord
      end
    end
@ -126,7 +126,7 @@ module Gitlab
    #
    # This method will return `nil` if no TTL could be obtained.
    def ttl
-      Gitlab::Redis::ClusterSharedState.with do |redis|
+      Gitlab::Redis::SharedState.with do |redis|
        ttl = redis.ttl(@redis_shared_state_key)

        ttl if ttl > 0
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@ -49524,9 +49524,15 @@ msgstr ""
 msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{count} %{issuables} have been included. Consider re-exporting with a narrower selection of %{issuables}."
 msgstr ""

+msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{count} projects have been included."
+msgstr ""
+
 msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{total_count} %{object_type} have been included. Consider re-exporting with a narrower selection of %{object_type}."
 msgstr ""

+msgid "This attachment has been truncated to avoid exceeding the maximum allowed attachment size of %{size_limit}. %{written_count} of %{total_count} projects have been included."
+msgstr ""
+
 msgid "This block is self-referential"
 msgstr ""

@ -52204,9 +52210,6 @@ msgstr ""
 msgid "UsageQuota|Filter projects data by month"
 msgstr ""

-msgid "UsageQuota|For more information about storage limits, see our %{faq_link_start}FAQ%{link_end}."
-msgstr ""
-
 msgid "UsageQuota|Git repository."
 msgstr ""

@ -56332,6 +56335,9 @@ msgstr ""
 msgid "Your Activity"
 msgstr ""

+msgid "Your CI runner usage CSV export containing the top %{exported_objects} has been added to this email as an attachment."
+msgstr ""
+
 msgid "Your CI/CD configuration syntax is invalid. Select the Validate tab for more details."
 msgstr ""

@ -56344,6 +56350,9 @@ msgstr ""
 msgid "Your CSV export of %{exported_objects} from project %{project_name} (%{project_url}) has been added to this email as an attachment."
 msgstr ""

+msgid "Your CSV export of the top %{exported_objects} has been added to this email as an attachment."
+msgstr ""
+
 msgid "Your CSV export request has succeeded. The result will be emailed to %{email}."
 msgstr ""

@ -56359,10 +56368,10 @@ msgstr ""
 msgid "Your DevOps Reports give an overview of how you are using GitLab from a feature perspective. Use them to view how you compare with other organizations, and how your teams compare against each other."
 msgstr ""

-msgid "Your Free top-level group, %{group_name}, has more than %{free_users_limit} users and uses more than %{free_storage_limit} of data. After usage limits are applied to Free top-level groups, projects in this group will be in a %{read_only_link_start}read-only state%{link_end}. To ensure that your group does not become read-only, you should contact a user with the Owner role for this group to upgrade to a paid tier, or manage your usage. For more information about the upcoming usage limits, see our %{faq_link_start}FAQ%{link_end}."
+msgid "Your Free top-level group, %{group_name}, has more than %{free_users_limit} users and uses more than %{free_storage_limit} of data. After usage limits are applied to Free top-level groups, projects in this group will be in a %{read_only_link_start}read-only state%{link_end}. To ensure that your group does not become read-only, you should contact a user with the Owner role for this group to upgrade to a paid tier, or manage your usage. %{faq_link_start}Learn more%{link_end} about the upcoming storage limits."
 msgstr ""

-msgid "Your Free top-level group, %{group_name}, has more than %{free_users_limit} users and uses more than %{free_storage_limit} of data. After usage limits are applied to Free top-level groups, projects in this group will be in a %{read_only_link_start}read-only state%{link_end}. You should reduce the number of users or upgrade to a paid tier %{strong_start}before%{strong_end} you manage your storage usage. Otherwise, your Free top-level group will become read-only immediately because the 5-user limit applies. For more information, see our %{faq_link_start}FAQ%{link_end}."
+msgid "Your Free top-level group, %{group_name}, has more than %{free_users_limit} users and uses more than %{free_storage_limit} of data. After usage limits are applied to Free top-level groups, projects in this group will be in a %{read_only_link_start}read-only state%{link_end}. You should reduce the number of users or upgrade to a paid tier %{strong_start}before%{strong_end} you manage your storage usage. Otherwise, your Free top-level group will become read-only immediately because the 5-user limit applies. %{faq_link_start}Learn more%{link_end} about namespace storage limits."
 msgstr ""

 msgid "Your GPG keys"
--- a/spec/frontend/vue_shared/issuable/list/components/issuable_list_root_spec.js
+++ b/spec/frontend/vue_shared/issuable/list/components/issuable_list_root_spec.js
@ -4,6 +4,7 @@ import VueDraggable from 'vuedraggable';

 import { nextTick } from 'vue';
 import { TEST_HOST } from 'helpers/test_constants';
+import { DRAG_DELAY } from '~/sortable/constants';

 import IssuableItem from '~/vue_shared/issuable/list/components/issuable_item.vue';
 import IssuableListRoot from '~/vue_shared/issuable/list/components/issuable_list_root.vue';
@ -476,6 +477,11 @@ describe('IssuableListRoot', () => {
        expect(findIssuableItem().classes()).toContain('gl-cursor-grab');
      });

+      it('sets delay and delayOnTouchOnly attributes on list', () => {
+        expect(findVueDraggable().vm.$attrs.delay).toBe(DRAG_DELAY);
+        expect(findVueDraggable().vm.$attrs.delayOnTouchOnly).toBe(true);
+      });
+
      it('emits a "reorder" event when user updates the issue order', () => {
        const oldIndex = 4;
        const newIndex = 6;
--- a/spec/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules_spec.rb
@ -12,10 +12,13 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::EvaluateWorkflowRules do
  end

  let(:step) { described_class.new(pipeline, command) }
+  let(:ff_always_set_pipeline_failure_reason) { true }

  describe '#perform!' do
    context 'when pipeline has been skipped by workflow configuration' do
      before do
+        stub_feature_flags(always_set_pipeline_failure_reason: ff_always_set_pipeline_failure_reason)
+
        allow(step).to receive(:workflow_rules_result)
          .and_return(
            double(pass?: false, variables: {})
@ -39,6 +42,20 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::EvaluateWorkflowRules do
      it 'saves workflow_rules_result' do
        expect(command.workflow_rules_result.variables).to eq({})
      end
+
+      it 'sets the failure reason', :aggregate_failures do
+        expect(pipeline).to be_failed
+        expect(pipeline).to be_filtered_by_workflow_rules
+      end
+
+      context 'when always_set_pipeline_failure_reason is disabled' do
+        let(:ff_always_set_pipeline_failure_reason) { false }
+
+        it 'does not set the failure reason', :aggregate_failures do
+          expect(pipeline).not_to be_failed
+          expect(pipeline.failure_reason).to be_blank
+        end
+      end
    end

    context 'when pipeline has not been skipped by workflow configuration' do
@ -67,6 +84,10 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::EvaluateWorkflowRules do
      it 'saves workflow_rules_result' do
        expect(command.workflow_rules_result.variables).to eq({ 'VAR1' => 'val2', 'VAR2' => 3 })
      end
+
+      it 'does not set a failure reason' do
+        expect(pipeline).not_to be_filtered_by_workflow_rules
+      end
    end
  end
 end
--- a/spec/lib/gitlab/ci/pipeline/chain/helpers_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/chain/helpers_spec.rb
@ -2,7 +2,7 @@

 require 'spec_helper'

-RSpec.describe Gitlab::Ci::Pipeline::Chain::Helpers do
+RSpec.describe Gitlab::Ci::Pipeline::Chain::Helpers, feature_category: :continuous_integration do
  let(:helper_class) do
    Class.new do
      include Gitlab::Ci::Pipeline::Chain::Helpers
@ -38,14 +38,35 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Helpers do
  describe '.error' do
    shared_examples 'error function' do
      specify do
-        expect(pipeline).to receive(:drop!).with(drop_reason).and_call_original
        expect(pipeline).to receive(:add_error_message).with(message).and_call_original
-        expect(pipeline).to receive(:ensure_project_iid!).twice.and_call_original
+
+        if command.save_incompleted
+          expect(pipeline).to receive(:ensure_project_iid!).twice.and_call_original
+          expect(pipeline).to receive(:drop!).with(drop_reason).and_call_original
+        end

        subject.error(message, config_error: config_error, drop_reason: drop_reason)

        expect(pipeline.yaml_errors).to eq(yaml_error)
        expect(pipeline.errors[:base]).to include(message)
+        expect(pipeline.status).to eq 'failed'
+        expect(pipeline.failure_reason).to eq drop_reason.to_s
+      end
+
+      context 'when feature flag always_set_pipeline_failure_reason is false' do
+        before do
+          stub_feature_flags(always_set_pipeline_failure_reason: false)
+        end
+
+        specify do
+          subject.error(message, config_error: config_error, drop_reason: drop_reason)
+
+          if command.save_incompleted
+            expect(pipeline.failure_reason).to eq drop_reason.to_s
+          else
+            expect(pipeline.failure_reason).not_to be_present
+          end
+        end
      end
    end

@ -79,6 +100,43 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Helpers do
          let(:yaml_error) { nil }

          it_behaves_like "error function"
+
+          specify do
+            subject.error(message, config_error: config_error, drop_reason: drop_reason)
+
+            expect(pipeline).to be_persisted
+          end
+
+          context ' when the drop reason is not persistable' do
+            let(:drop_reason) { :filtered_by_rules }
+            let(:command) { double(project: nil) }
+
+            specify do
+              expect(command).to receive(:increment_pipeline_failure_reason_counter)
+
+              subject.error(message, config_error: config_error, drop_reason: drop_reason)
+
+              expect(pipeline).to be_failed
+              expect(pipeline.failure_reason).to eq drop_reason.to_s
+              expect(pipeline).not_to be_persisted
+            end
+          end
+
+          context 'when save_incompleted is false' do
+            let(:command) { double(save_incompleted: false, project: nil) }
+
+            before do
+              allow(command).to receive(:increment_pipeline_failure_reason_counter)
+            end
+
+            it_behaves_like "error function"
+
+            specify do
+              subject.error(message, config_error: config_error, drop_reason: drop_reason)
+
+              expect(pipeline).not_to be_persisted
+            end
+          end
        end
      end
    end
--- a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb
@ -34,12 +34,15 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Populate, feature_category: :continu
    { rspec: { script: 'rspec' } }
  end

+  let(:ff_always_set_pipeline_failure_reason) { true }
+
  def run_chain
    dependencies.map(&:perform!)
    step.perform!
  end

  before do
+    stub_feature_flags(always_set_pipeline_failure_reason: ff_always_set_pipeline_failure_reason)
    stub_ci_pipeline_yaml_file(YAML.dump(config))
  end

@ -100,7 +103,27 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Populate, feature_category: :continu

    it 'increments the error metric' do
      counter = Gitlab::Metrics.counter(:gitlab_ci_pipeline_failure_reasons, 'desc')
-      expect { run_chain }.to change { counter.get(reason: 'unknown_failure') }.by(1)
+      expect { run_chain }.to change { counter.get(reason: 'filtered_by_rules') }.by(1)
+    end
+
+    it 'sets the failure reason without persisting the pipeline', :aggregate_failures do
+      run_chain
+
+      expect(pipeline).not_to be_persisted
+      expect(pipeline).to be_failed
+      expect(pipeline).to be_filtered_by_rules
+    end
+
+    context 'when ff always_set_pipeline_failure_reason is disabled' do
+      let(:ff_always_set_pipeline_failure_reason) { false }
+
+      it 'sets the failure reason without persisting the pipeline', :aggregate_failures do
+        run_chain
+
+        expect(pipeline).not_to be_persisted
+        expect(pipeline).not_to be_failed
+        expect(pipeline).not_to be_filtered_by_rules
+      end
    end
  end

--- a/spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb
@ -2,7 +2,7 @@

 require 'spec_helper'

-RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::External do
+RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::External, feature_category: :continuous_integration do
  let_it_be(:project) { create(:project) }
  let_it_be(:user) { create(:user, :with_sign_ins) }

@ -328,11 +328,12 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::External do
      context 'when save_incompleted is false' do
        let(:save_incompleted) { false }

-        it 'adds errors to the pipeline without dropping it' do
+        it 'adds errors to the pipeline without persisting it', :aggregate_failures do
          perform!

-          expect(pipeline.status).to eq('pending')
          expect(pipeline).not_to be_persisted
+          expect(pipeline.status).to eq('failed')
+          expect(pipeline).to be_external_validation_failure
          expect(pipeline.errors.to_a).to include('External validation failed')
        end

--- a/spec/lib/gitlab/exclusive_lease_spec.rb
+++ b/spec/lib/gitlab/exclusive_lease_spec.rb
@ -3,7 +3,7 @@
 require 'spec_helper'

 RSpec.describe Gitlab::ExclusiveLease, :request_store,
-  :clean_gitlab_redis_cluster_shared_state, feature_category: :shared do
+  :clean_gitlab_redis_shared_state, feature_category: :shared do
  let(:unique_key) { SecureRandom.hex(10) }

  describe '#try_obtain' do
--- a/spec/lib/gitlab/sidekiq_status_spec.rb
+++ b/spec/lib/gitlab/sidekiq_status_spec.rb
@ -3,7 +3,7 @@
 require 'spec_helper'

 RSpec.describe Gitlab::SidekiqStatus, :clean_gitlab_redis_queues,
-  :clean_gitlab_redis_shared_state, :clean_gitlab_redis_cluster_shared_state do
+  :clean_gitlab_redis_shared_state do
  shared_examples 'tracking status in redis' do
    describe '.set' do
      it 'stores the job ID' do
--- a/spec/models/ci/pipeline_spec.rb
+++ b/spec/models/ci/pipeline_spec.rb
@ -3794,6 +3794,44 @@ RSpec.describe Ci::Pipeline, :mailer, factory_default: :keep, feature_category:
    end
  end

+  describe '#set_failed' do
+    let(:pipeline) { build(:ci_pipeline) }
+
+    it 'marks the pipeline as failed with the given reason without saving', :aggregate_failures do
+      pipeline.set_failed(:filtered_by_rules)
+
+      expect(pipeline).to be_failed
+      expect(pipeline).to be_filtered_by_rules
+      expect(pipeline).not_to be_persisted
+    end
+  end
+
+  describe '#filtered_as_empty?' do
+    let(:pipeline) { build_stubbed(:ci_pipeline) }
+
+    subject { pipeline.filtered_as_empty? }
+
+    it { is_expected.to eq false }
+
+    context 'when the pipeline is failed' do
+      using RSpec::Parameterized::TableSyntax
+
+      where(:drop_reason, :expected) do
+        :unknown_failure            | false
+        :filtered_by_rules          | true
+        :filtered_by_workflow_rules | true
+      end
+
+      with_them do
+        before do
+          pipeline.set_failed(drop_reason)
+        end
+
+        it { is_expected.to eq expected }
+      end
+    end
+  end
+
  describe '#has_yaml_errors?' do
    let(:pipeline) { build_stubbed(:ci_pipeline) }

--- a/spec/models/service_desk/custom_email_credential_spec.rb
+++ b/spec/models/service_desk/custom_email_credential_spec.rb
@ -21,7 +21,7 @@ RSpec.describe ServiceDesk::CustomEmailCredential, feature_category: :service_de
    it { is_expected.not_to allow_value('/example').for(:smtp_address) }
    it { is_expected.not_to allow_value('localhost').for(:smtp_address) }
    it { is_expected.not_to allow_value('127.0.0.1').for(:smtp_address) }
-    it { is_expected.not_to allow_value('192.168.12.12').for(:smtp_address) } # disallow local network
+    it { is_expected.to allow_value('192.168.12.12').for(:smtp_address) } # allow local network on self-managed

    it { is_expected.to validate_presence_of(:smtp_port) }
    it { is_expected.to validate_numericality_of(:smtp_port).only_integer.is_greater_than(0) }
@ -31,6 +31,10 @@ RSpec.describe ServiceDesk::CustomEmailCredential, feature_category: :service_de

    it { is_expected.to validate_presence_of(:smtp_password) }
    it { is_expected.to validate_length_of(:smtp_password).is_at_least(8).is_at_most(128) }
+
+    context 'when SaaS', :saas do
+      it { is_expected.not_to allow_value('192.168.12.12').for(:smtp_address) } # Disallow local network on .com
+    end
  end

  describe 'encrypted #smtp_username' do
--- a/spec/workers/background_migration/ci_database_worker_spec.rb
+++ b/spec/workers/background_migration/ci_database_worker_spec.rb
@ -2,7 +2,7 @@

 require 'spec_helper'

-RSpec.describe BackgroundMigration::CiDatabaseWorker, :clean_gitlab_redis_cluster_shared_state,
+RSpec.describe BackgroundMigration::CiDatabaseWorker, :clean_gitlab_redis_shared_state,
  feature_category: :database do
  before do
    skip_if_shared_database(:ci)
--- a/spec/workers/background_migration_worker_spec.rb
+++ b/spec/workers/background_migration_worker_spec.rb
@ -2,7 +2,7 @@

 require 'spec_helper'

-RSpec.describe BackgroundMigrationWorker, :clean_gitlab_redis_cluster_shared_state,
+RSpec.describe BackgroundMigrationWorker, :clean_gitlab_redis_shared_state,
  feature_category: :database do
  it_behaves_like 'it runs background migration jobs', 'main'
 end
--- a/spec/workers/every_sidekiq_worker_spec.rb
+++ b/spec/workers/every_sidekiq_worker_spec.rb
@ -475,7 +475,8 @@ RSpec.describe 'Every Sidekiq worker', feature_category: :shared do
        'Zoekt::IndexerWorker' => 2,
        'Issuable::RelatedLinksCreateWorker' => 3,
        'BulkImports::RelationBatchExportWorker' => 6,
-        'BulkImports::RelationExportWorker' => 6
+        'BulkImports::RelationExportWorker' => 6,
+        'Ci::Runners::ExportUsageCsvWorker' => 3
      }.merge(extra_retry_exceptions)
    end

--- a/spec/workers/integrations/slack_event_worker_spec.rb
+++ b/spec/workers/integrations/slack_event_worker_spec.rb
@ -2,7 +2,7 @@

 require 'spec_helper'

-RSpec.describe Integrations::SlackEventWorker, :clean_gitlab_redis_cluster_shared_state,
+RSpec.describe Integrations::SlackEventWorker, :clean_gitlab_redis_shared_state,
  feature_category: :integrations do
  describe '.event?' do
    subject { described_class.event?(event) }