Add latest changes from gitlab-org/gitlab@master

Gemfile

@@ -578,7 +578,7 @@ group :test do
   gem 'webmock', '~> 3.23.0', feature_category: :shared
   gem 'rails-controller-testing' # rubocop:todo Gemfile/MissingFeatureCategory
   gem 'concurrent-ruby', '~> 1.1' # rubocop:todo Gemfile/MissingFeatureCategory
-  gem 'test-prof', '~> 1.3.3', feature_category: :tooling
+  gem 'test-prof', '~> 1.4.0', feature_category: :tooling
   gem 'rspec_junit_formatter' # rubocop:todo Gemfile/MissingFeatureCategory
   gem 'guard-rspec' # rubocop:todo Gemfile/MissingFeatureCategory
   gem 'axe-core-rspec', '~> 4.9.0', feature_category: :tooling

Gemfile.checksum

@@ -702,7 +702,7 @@
 {"name":"term-ansicolor","version":"1.7.1","platform":"ruby","checksum":"92339ffec77c4bddc786a29385c91601dd52fc68feda23609bba0491229b05f7"},
 {"name":"terminal-table","version":"3.0.2","platform":"ruby","checksum":"f951b6af5f3e00203fb290a669e0a85c5dd5b051b3b023392ccfd67ba5abae91"},
 {"name":"terser","version":"1.0.2","platform":"ruby","checksum":"80c2e0bc7e2db4e12e8529658f9e0820e13d685ae67d745bf981f269743bb28e"},
-{"name":"test-prof","version":"1.3.3.1","platform":"ruby","checksum":"0eb5dfdd6c2f70a8f1402683793e60b3ff63582a31690dfd8ef0aefd2c99b153"},
+{"name":"test-prof","version":"1.4.0","platform":"ruby","checksum":"966bc3efc37216e9e79b44274e7f74e3c6614b3dba7fba2f5ad326ab90910f74"},
 {"name":"test_file_finder","version":"0.3.1","platform":"ruby","checksum":"83fb0588a06b2784b51892910b9bfd06609f8d31f2d851a98d976f644d177199"},
 {"name":"text","version":"1.3.1","platform":"ruby","checksum":"2fbbbc82c1ce79c4195b13018a87cbb00d762bda39241bb3cdc32792759dd3f4"},
 {"name":"thor","version":"1.3.1","platform":"ruby","checksum":"fa7e3471d4f6a27138e3d9c9b0d4daac9c3d7383927667ae83e9ab42ae7401ef"},

Gemfile.lock

@@ -1809,7 +1809,7 @@ GEM
       unicode-display_width (>= 1.1.1, < 3)
     terser (1.0.2)
       execjs (>= 0.3.0, < 3)
-    test-prof (1.3.3.1)
+    test-prof (1.4.0)
     test_file_finder (0.3.1)
       faraday (>= 1.0, < 3.0, != 2.0.0)
     text (1.3.1)
@@ -2289,7 +2289,7 @@ DEPENDENCIES
   tanuki_emoji (~> 0.9)
   telesignenterprise (~> 2.2)
   terser (= 1.0.2)
-  test-prof (~> 1.3.3)
+  test-prof (~> 1.4.0)
   test_file_finder (~> 0.3.1)
   thrift (>= 0.16.0)
   timfel-krb5-auth (~> 0.8)

Gemfile.next.checksum

@@ -713,7 +713,7 @@
 {"name":"term-ansicolor","version":"1.7.1","platform":"ruby","checksum":"92339ffec77c4bddc786a29385c91601dd52fc68feda23609bba0491229b05f7"},
 {"name":"terminal-table","version":"3.0.2","platform":"ruby","checksum":"f951b6af5f3e00203fb290a669e0a85c5dd5b051b3b023392ccfd67ba5abae91"},
 {"name":"terser","version":"1.0.2","platform":"ruby","checksum":"80c2e0bc7e2db4e12e8529658f9e0820e13d685ae67d745bf981f269743bb28e"},
-{"name":"test-prof","version":"1.3.3.1","platform":"ruby","checksum":"0eb5dfdd6c2f70a8f1402683793e60b3ff63582a31690dfd8ef0aefd2c99b153"},
+{"name":"test-prof","version":"1.4.0","platform":"ruby","checksum":"966bc3efc37216e9e79b44274e7f74e3c6614b3dba7fba2f5ad326ab90910f74"},
 {"name":"test_file_finder","version":"0.3.1","platform":"ruby","checksum":"83fb0588a06b2784b51892910b9bfd06609f8d31f2d851a98d976f644d177199"},
 {"name":"text","version":"1.3.1","platform":"ruby","checksum":"2fbbbc82c1ce79c4195b13018a87cbb00d762bda39241bb3cdc32792759dd3f4"},
 {"name":"thor","version":"1.3.1","platform":"ruby","checksum":"fa7e3471d4f6a27138e3d9c9b0d4daac9c3d7383927667ae83e9ab42ae7401ef"},

Gemfile.next.lock

@@ -1824,7 +1824,7 @@ GEM
       unicode-display_width (>= 1.1.1, < 3)
     terser (1.0.2)
       execjs (>= 0.3.0, < 3)
-    test-prof (1.3.3.1)
+    test-prof (1.4.0)
     test_file_finder (0.3.1)
       faraday (>= 1.0, < 3.0, != 2.0.0)
     text (1.3.1)
@@ -2304,7 +2304,7 @@ DEPENDENCIES
   tanuki_emoji (~> 0.9)
   telesignenterprise (~> 2.2)
   terser (= 1.0.2)
-  test-prof (~> 1.3.3)
+  test-prof (~> 1.4.0)
   test_file_finder (~> 0.3.1)
   thrift (>= 0.16.0)
   timfel-krb5-auth (~> 0.8)

config/initializers/7_redis.rb

@@ -29,6 +29,9 @@ if Rails.application.config.cache_classes
   # this only instruments `RedisClient` used in `Sidekiq.redis`
   RedisClient.register(Gitlab::Instrumentation::RedisClientMiddleware)
   RedisClient.prepend(Gitlab::Patch::RedisClient)
+
+  # This specifically instruments for Redis Cluster node failures.
+  RedisClient::Cluster::Router.prepend(Gitlab::Instrumentation::RedisClusterRouter)
 end
 
 if Gitlab::Redis::Workhorse.params[:cluster].present?

doc/user/application_security/secret_detection/pipeline/index.md

@@ -22,6 +22,7 @@ With GitLab Ultimate, pipeline secret detection results are also processed so yo
 - Use them in approval workflows.
 - Review them in the security dashboard.
 - [Automatically respond](../automatic_response.md) to leaks in public repositories.
+- Enforce consistent secret detection rules across projects using [security policies](../../policies/index.md).
 
 <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For an interactive reading and how-to demo of this pipeline secret detection documentation see:
 
@@ -155,6 +156,7 @@ Different features are available in different [GitLab tiers](https://about.gitla
 | [Manage vulnerabilities](../../vulnerability_report/index.md)                                        | **{dotted-circle}** No | **{check-circle}** Yes |
 | [Access the Security Dashboard](../../security_dashboard/index.md)                                   | **{dotted-circle}** No | **{check-circle}** Yes |
 | [Customize analyzer rulesets](#customize-analyzer-rulesets)                                          | **{dotted-circle}** No | **{check-circle}** Yes |
+| [Enable security policies](../../policies/index.md)                                                  | **{dotted-circle}** No | **{check-circle}** Yes |
 
 ### Enable the analyzer
 

lib/gitlab/diff/char_diff.rb

@@ -47,7 +47,10 @@ module Gitlab
 
       def to_html
         @changes.map do |op, text|
-          %(<span class="#{html_class_names(op)}">#{ERB::Util.html_escape(text)}</span>)
+          text = ERB::Util.html_escape(text)
+          text.gsub!("\n", "↵\n") if op == :insert || op == :delete
+
+          %(<span class="#{html_class_names(op)}">#{text}</span>)
         end.join.html_safe
       end
 

lib/gitlab/instrumentation/redis_cluster_router.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Instrumentation
+    module RedisClusterRouter
+      # Patch the `send_command` method in RedisClient::Cluster::Router
+      # See https://github.com/redis-rb/redis-cluster-client/blob/v0.8.2/lib/redis_client/cluster/router.rb#L34
+      #
+      # When a Redis Cluster is in a fail state, we might not have metrics on the server-side.
+      # This allows the application to dump its local topology state to get the client-side perspective of any
+      # cluster failure.
+      def send_command(method, command, *args, &block)
+        super
+      rescue ::RedisClient::Cluster::NodeMightBeDown => e
+        # rubocop:disable Gitlab/ModuleWithInstanceVariables -- this class is used to monkeypatch RedisClient::Cluster::Router
+        slots_map = Gitlab::Instrumentation::RedisClusterRouter.format_slotmap(@node.instance_variable_get(:@slots))
+        Gitlab::ErrorTracking.log_exception(
+          e,
+          node_keys: @node.node_keys,
+          slots_map: slots_map
+        )
+
+        inst = instrumentation_class(@config)
+        inst.instance_count_exception(e) if inst
+        # rubocop:enable Gitlab/ModuleWithInstanceVariables
+
+        raise e
+      end
+
+      private
+
+      def instrumentation_class(config)
+        name = config.client_config.dig(:custom, :instrumentation_class)
+        return unless name
+
+        ::Gitlab::Instrumentation::Redis.storage_hash[name]
+      end
+
+      class << self
+        def format_slotmap(slots)
+          return {} unless slots
+
+          slotmap = {}
+          (0..16383).each do |c|
+            node_key = slots[c]
+            next unless node_key
+
+            slotmap[node_key] ||= []
+            slotmap[node_key] << c
+          end
+
+          slotmap.transform_values { |v| compact_array(v) }
+        end
+
+        # compact_array converts an array of integers into a range string
+        # e.g. [0, 1, 2, 4, 5, 6] to "0-2,4-6"
+        def compact_array(arr)
+          return "" if arr.empty?
+
+          range = ""
+          prev = nil
+          arr.each do |i|
+            if prev.nil?
+              range += i.to_s
+            elsif prev + 1 < i
+              range += "-#{prev},#{i}"
+            end
+
+            prev = i
+          end
+          range += "-#{prev}"
+
+          range
+        end
+      end
+    end
+  end
+end

qa/Gemfile

@@ -19,7 +19,7 @@ gem 'faker', '~> 3.4', '>= 3.4.2'
 gem 'knapsack', '~> 4.0'
 gem 'parallel_tests', '~> 4.7', '>= 4.7.1'
 gem 'rotp', '~> 6.3.0'
-gem 'parallel', '~> 1.26', '>= 1.26.1'
+gem 'parallel', '~> 1.26', '>= 1.26.2'
 gem 'rainbow', '~> 3.1.1'
 gem 'rspec-parameterized', '~> 1.0.2'
 gem 'octokit', '~> 9.1.0', require: false

qa/Gemfile.lock

@@ -246,7 +246,7 @@ GEM
       faraday (>= 1, < 3)
       sawyer (~> 0.9)
     os (1.1.4)
-    parallel (1.26.1)
+    parallel (1.26.2)
     parallel_tests (4.7.1)
       parallel
     parser (3.3.3.0)
@@ -412,7 +412,7 @@ DEPENDENCIES
   knapsack (~> 4.0)
   nokogiri (~> 1.16, >= 1.16.7)
   octokit (~> 9.1.0)
-  parallel (~> 1.26, >= 1.26.1)
+  parallel (~> 1.26, >= 1.26.2)
   parallel_tests (~> 4.7, >= 4.7.1)
   pry-byebug (~> 3.10.1)
   rainbow (~> 3.1.1)

spec/lib/gitlab/diff/char_diff_spec.rb

@@ -3,7 +3,7 @@
 require 'fast_spec_helper'
 require 'diff_match_patch'
 
-RSpec.describe Gitlab::Diff::CharDiff do
+RSpec.describe Gitlab::Diff::CharDiff, feature_category: :product_planning do
   let(:old_string) { "Helo \n Worlld" }
   let(:new_string) { "Hello \n World" }
 
@@ -75,5 +75,22 @@ RSpec.describe Gitlab::Diff::CharDiff do
         '<span class="idiff">d</span>'
       )
     end
+
+    context 'when changes involve newlines' do
+      let(:old_string) { "Hello\nWorld" }
+      let(:new_string) { "Hello World\n" }
+
+      it 'replaces newlines with ↵' do
+        subject.generate_diff
+
+        expect(subject.to_html).to eq(
+          '<span class="idiff">Hello</span>' \
+          "<span class=\"idiff deletion\">↵\n</span>" \
+          '<span class="idiff addition"> </span>' \
+          '<span class="idiff">World</span>' \
+          "<span class=\"idiff addition\">↵\n</span>"
+        )
+      end
+    end
   end
 end

spec/lib/gitlab/instrumentation/redis_cluster_router_spec.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'rspec-parameterized'
+
+RSpec.describe Gitlab::Instrumentation::RedisClusterRouter, feature_category: :redis do
+  describe '#send_command', if: ::Gitlab::Redis::Cache.params[:nodes] do
+    before do
+      Gitlab::Redis::Cache.with do |c|
+        allow(c._client.instance_variable_get(:@router))
+          .to receive(:assign_node)
+          .and_raise(::RedisClient::Cluster::NodeMightBeDown)
+      end
+    end
+
+    it 'tracks exception' do
+      expect(Gitlab::ErrorTracking)
+        .to receive(:log_exception).with(
+          instance_of(::RedisClient::Cluster::NodeMightBeDown), node_keys: anything, slots_map: anything
+        )
+
+      expect(Gitlab::Instrumentation::Redis::Cache)
+        .to receive(:instance_count_exception).with(instance_of(::RedisClient::Cluster::NodeMightBeDown))
+
+      expect do
+        Gitlab::Redis::Cache.with { |c| c.decr('test') }
+      end.to raise_error(Redis::Cluster::NodeMightBeDown) # redis-rb gem intercepts and replaces error
+    end
+  end
+
+  describe '.format_slotmap' do
+    it 'handles empty slot array' do
+      expect(described_class.format_slotmap([])).to eq({})
+    end
+
+    it 'handles incomplete slot array' do
+      input = ['localhost:1', 'localhost:1', nil, 'localhost:2']
+      expect(described_class.format_slotmap(input)).to eq({ 'localhost:1' => '0-1', 'localhost:2' => '3-3' })
+    end
+
+    it 'handles complete slot array' do
+      input = (['localhost:1'] * 5000) + (['localhost:2'] * 5000) + (['localhost:3'] * 5000) + (['localhost:1'] * 5000)
+      expect(described_class.format_slotmap(input))
+        .to eq({
+          'localhost:1' => '0-4999,15000-16383', 'localhost:2' => '5000-9999', 'localhost:3' => '10000-14999'
+        })
+    end
+  end
+
+  describe '.compact_array' do
+    using RSpec::Parameterized::TableSyntax
+
+    where(:input, :output) do
+      [1, 2, 3, 4, 5, 6, 7] | "1-7" # contiguous
+      [1, 2, 3, 5, 6, 7, 9, 10] | "1-3,5-7,9-10" # non-contiguous
+      [1, 2, 3, 4, 5, 7] | "1-5,7-7" # contigious with 1 ending slot
+      [] | "" # empty
+      [1, 1, 1, 1] | "1-1" # homogenuous array
+    end
+
+    with_them do
+      it do
+        expect(described_class.compact_array(input)).to eq(output)
+      end
+    end
+  end
+end

spec/support/factory_default.rb

@@ -10,7 +10,7 @@ module Gitlab
   end
 end
 
-TestProf::FactoryDefault::DefaultSyntax.prepend Gitlab::FreezeFactoryDefault
+TestProf::FactoryDefault::FactoryBotPatch::SyntaxExt.prepend Gitlab::FreezeFactoryDefault
 
 RSpec.configure do |config|
   config.after do |ex|