diff --git a/Gemfile b/Gemfile index f29609834fd..4a13d3bb263 100644 --- a/Gemfile +++ b/Gemfile @@ -578,7 +578,7 @@ group :test do gem 'webmock', '~> 3.23.0', feature_category: :shared gem 'rails-controller-testing' # rubocop:todo Gemfile/MissingFeatureCategory gem 'concurrent-ruby', '~> 1.1' # rubocop:todo Gemfile/MissingFeatureCategory - gem 'test-prof', '~> 1.3.3', feature_category: :tooling + gem 'test-prof', '~> 1.4.0', feature_category: :tooling gem 'rspec_junit_formatter' # rubocop:todo Gemfile/MissingFeatureCategory gem 'guard-rspec' # rubocop:todo Gemfile/MissingFeatureCategory gem 'axe-core-rspec', '~> 4.9.0', feature_category: :tooling diff --git a/Gemfile.checksum b/Gemfile.checksum index 22eebc3ff0a..0fc69cc9ccd 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -702,7 +702,7 @@ {"name":"term-ansicolor","version":"1.7.1","platform":"ruby","checksum":"92339ffec77c4bddc786a29385c91601dd52fc68feda23609bba0491229b05f7"}, {"name":"terminal-table","version":"3.0.2","platform":"ruby","checksum":"f951b6af5f3e00203fb290a669e0a85c5dd5b051b3b023392ccfd67ba5abae91"}, {"name":"terser","version":"1.0.2","platform":"ruby","checksum":"80c2e0bc7e2db4e12e8529658f9e0820e13d685ae67d745bf981f269743bb28e"}, -{"name":"test-prof","version":"1.3.3.1","platform":"ruby","checksum":"0eb5dfdd6c2f70a8f1402683793e60b3ff63582a31690dfd8ef0aefd2c99b153"}, +{"name":"test-prof","version":"1.4.0","platform":"ruby","checksum":"966bc3efc37216e9e79b44274e7f74e3c6614b3dba7fba2f5ad326ab90910f74"}, {"name":"test_file_finder","version":"0.3.1","platform":"ruby","checksum":"83fb0588a06b2784b51892910b9bfd06609f8d31f2d851a98d976f644d177199"}, {"name":"text","version":"1.3.1","platform":"ruby","checksum":"2fbbbc82c1ce79c4195b13018a87cbb00d762bda39241bb3cdc32792759dd3f4"}, {"name":"thor","version":"1.3.1","platform":"ruby","checksum":"fa7e3471d4f6a27138e3d9c9b0d4daac9c3d7383927667ae83e9ab42ae7401ef"}, diff --git a/Gemfile.lock b/Gemfile.lock index f81db106342..37596c0a28a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1809,7 +1809,7 @@ GEM unicode-display_width (>= 1.1.1, < 3) terser (1.0.2) execjs (>= 0.3.0, < 3) - test-prof (1.3.3.1) + test-prof (1.4.0) test_file_finder (0.3.1) faraday (>= 1.0, < 3.0, != 2.0.0) text (1.3.1) @@ -2289,7 +2289,7 @@ DEPENDENCIES tanuki_emoji (~> 0.9) telesignenterprise (~> 2.2) terser (= 1.0.2) - test-prof (~> 1.3.3) + test-prof (~> 1.4.0) test_file_finder (~> 0.3.1) thrift (>= 0.16.0) timfel-krb5-auth (~> 0.8) diff --git a/Gemfile.next.checksum b/Gemfile.next.checksum index 67e31e8859c..042ffc8a383 100644 --- a/Gemfile.next.checksum +++ b/Gemfile.next.checksum @@ -713,7 +713,7 @@ {"name":"term-ansicolor","version":"1.7.1","platform":"ruby","checksum":"92339ffec77c4bddc786a29385c91601dd52fc68feda23609bba0491229b05f7"}, {"name":"terminal-table","version":"3.0.2","platform":"ruby","checksum":"f951b6af5f3e00203fb290a669e0a85c5dd5b051b3b023392ccfd67ba5abae91"}, {"name":"terser","version":"1.0.2","platform":"ruby","checksum":"80c2e0bc7e2db4e12e8529658f9e0820e13d685ae67d745bf981f269743bb28e"}, -{"name":"test-prof","version":"1.3.3.1","platform":"ruby","checksum":"0eb5dfdd6c2f70a8f1402683793e60b3ff63582a31690dfd8ef0aefd2c99b153"}, +{"name":"test-prof","version":"1.4.0","platform":"ruby","checksum":"966bc3efc37216e9e79b44274e7f74e3c6614b3dba7fba2f5ad326ab90910f74"}, {"name":"test_file_finder","version":"0.3.1","platform":"ruby","checksum":"83fb0588a06b2784b51892910b9bfd06609f8d31f2d851a98d976f644d177199"}, {"name":"text","version":"1.3.1","platform":"ruby","checksum":"2fbbbc82c1ce79c4195b13018a87cbb00d762bda39241bb3cdc32792759dd3f4"}, {"name":"thor","version":"1.3.1","platform":"ruby","checksum":"fa7e3471d4f6a27138e3d9c9b0d4daac9c3d7383927667ae83e9ab42ae7401ef"}, diff --git a/Gemfile.next.lock b/Gemfile.next.lock index 7c6287978b4..668bb517755 100644 --- a/Gemfile.next.lock +++ b/Gemfile.next.lock @@ -1824,7 +1824,7 @@ GEM unicode-display_width (>= 1.1.1, < 3) terser (1.0.2) execjs (>= 0.3.0, < 3) - test-prof (1.3.3.1) + test-prof (1.4.0) test_file_finder (0.3.1) faraday (>= 1.0, < 3.0, != 2.0.0) text (1.3.1) @@ -2304,7 +2304,7 @@ DEPENDENCIES tanuki_emoji (~> 0.9) telesignenterprise (~> 2.2) terser (= 1.0.2) - test-prof (~> 1.3.3) + test-prof (~> 1.4.0) test_file_finder (~> 0.3.1) thrift (>= 0.16.0) timfel-krb5-auth (~> 0.8) diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb index 521e6237f4c..c0f8dee2f10 100644 --- a/config/initializers/7_redis.rb +++ b/config/initializers/7_redis.rb @@ -29,6 +29,9 @@ if Rails.application.config.cache_classes # this only instruments `RedisClient` used in `Sidekiq.redis` RedisClient.register(Gitlab::Instrumentation::RedisClientMiddleware) RedisClient.prepend(Gitlab::Patch::RedisClient) + + # This specifically instruments for Redis Cluster node failures. + RedisClient::Cluster::Router.prepend(Gitlab::Instrumentation::RedisClusterRouter) end if Gitlab::Redis::Workhorse.params[:cluster].present? diff --git a/doc/user/application_security/secret_detection/pipeline/index.md b/doc/user/application_security/secret_detection/pipeline/index.md index 7c4c21b0ffe..2f804dc2a29 100644 --- a/doc/user/application_security/secret_detection/pipeline/index.md +++ b/doc/user/application_security/secret_detection/pipeline/index.md @@ -22,6 +22,7 @@ With GitLab Ultimate, pipeline secret detection results are also processed so yo - Use them in approval workflows. - Review them in the security dashboard. - [Automatically respond](../automatic_response.md) to leaks in public repositories. +- Enforce consistent secret detection rules across projects using [security policies](../../policies/index.md). For an interactive reading and how-to demo of this pipeline secret detection documentation see: @@ -155,6 +156,7 @@ Different features are available in different [GitLab tiers](https://about.gitla | [Manage vulnerabilities](../../vulnerability_report/index.md) | **{dotted-circle}** No | **{check-circle}** Yes | | [Access the Security Dashboard](../../security_dashboard/index.md) | **{dotted-circle}** No | **{check-circle}** Yes | | [Customize analyzer rulesets](#customize-analyzer-rulesets) | **{dotted-circle}** No | **{check-circle}** Yes | +| [Enable security policies](../../policies/index.md) | **{dotted-circle}** No | **{check-circle}** Yes | ### Enable the analyzer diff --git a/lib/gitlab/diff/char_diff.rb b/lib/gitlab/diff/char_diff.rb index d686c43185b..a659e4fe759 100644 --- a/lib/gitlab/diff/char_diff.rb +++ b/lib/gitlab/diff/char_diff.rb @@ -47,7 +47,10 @@ module Gitlab def to_html @changes.map do |op, text| - %(#{ERB::Util.html_escape(text)}) + text = ERB::Util.html_escape(text) + text.gsub!("\n", "↵\n") if op == :insert || op == :delete + + %(#{text}) end.join.html_safe end diff --git a/lib/gitlab/instrumentation/redis_cluster_router.rb b/lib/gitlab/instrumentation/redis_cluster_router.rb new file mode 100644 index 00000000000..8d7cd073542 --- /dev/null +++ b/lib/gitlab/instrumentation/redis_cluster_router.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module Instrumentation + module RedisClusterRouter + # Patch the `send_command` method in RedisClient::Cluster::Router + # See https://github.com/redis-rb/redis-cluster-client/blob/v0.8.2/lib/redis_client/cluster/router.rb#L34 + # + # When a Redis Cluster is in a fail state, we might not have metrics on the server-side. + # This allows the application to dump its local topology state to get the client-side perspective of any + # cluster failure. + def send_command(method, command, *args, &block) + super + rescue ::RedisClient::Cluster::NodeMightBeDown => e + # rubocop:disable Gitlab/ModuleWithInstanceVariables -- this class is used to monkeypatch RedisClient::Cluster::Router + slots_map = Gitlab::Instrumentation::RedisClusterRouter.format_slotmap(@node.instance_variable_get(:@slots)) + Gitlab::ErrorTracking.log_exception( + e, + node_keys: @node.node_keys, + slots_map: slots_map + ) + + inst = instrumentation_class(@config) + inst.instance_count_exception(e) if inst + # rubocop:enable Gitlab/ModuleWithInstanceVariables + + raise e + end + + private + + def instrumentation_class(config) + name = config.client_config.dig(:custom, :instrumentation_class) + return unless name + + ::Gitlab::Instrumentation::Redis.storage_hash[name] + end + + class << self + def format_slotmap(slots) + return {} unless slots + + slotmap = {} + (0..16383).each do |c| + node_key = slots[c] + next unless node_key + + slotmap[node_key] ||= [] + slotmap[node_key] << c + end + + slotmap.transform_values { |v| compact_array(v) } + end + + # compact_array converts an array of integers into a range string + # e.g. [0, 1, 2, 4, 5, 6] to "0-2,4-6" + def compact_array(arr) + return "" if arr.empty? + + range = "" + prev = nil + arr.each do |i| + if prev.nil? + range += i.to_s + elsif prev + 1 < i + range += "-#{prev},#{i}" + end + + prev = i + end + range += "-#{prev}" + + range + end + end + end + end +end diff --git a/qa/Gemfile b/qa/Gemfile index bc070f21faa..cbd1411a776 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -19,7 +19,7 @@ gem 'faker', '~> 3.4', '>= 3.4.2' gem 'knapsack', '~> 4.0' gem 'parallel_tests', '~> 4.7', '>= 4.7.1' gem 'rotp', '~> 6.3.0' -gem 'parallel', '~> 1.26', '>= 1.26.1' +gem 'parallel', '~> 1.26', '>= 1.26.2' gem 'rainbow', '~> 3.1.1' gem 'rspec-parameterized', '~> 1.0.2' gem 'octokit', '~> 9.1.0', require: false diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index d6da265f9bc..24e940b9737 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -246,7 +246,7 @@ GEM faraday (>= 1, < 3) sawyer (~> 0.9) os (1.1.4) - parallel (1.26.1) + parallel (1.26.2) parallel_tests (4.7.1) parallel parser (3.3.3.0) @@ -412,7 +412,7 @@ DEPENDENCIES knapsack (~> 4.0) nokogiri (~> 1.16, >= 1.16.7) octokit (~> 9.1.0) - parallel (~> 1.26, >= 1.26.1) + parallel (~> 1.26, >= 1.26.2) parallel_tests (~> 4.7, >= 4.7.1) pry-byebug (~> 3.10.1) rainbow (~> 3.1.1) diff --git a/spec/lib/gitlab/diff/char_diff_spec.rb b/spec/lib/gitlab/diff/char_diff_spec.rb index ca0ed6e840d..ae7adc4c759 100644 --- a/spec/lib/gitlab/diff/char_diff_spec.rb +++ b/spec/lib/gitlab/diff/char_diff_spec.rb @@ -3,7 +3,7 @@ require 'fast_spec_helper' require 'diff_match_patch' -RSpec.describe Gitlab::Diff::CharDiff do +RSpec.describe Gitlab::Diff::CharDiff, feature_category: :product_planning do let(:old_string) { "Helo \n Worlld" } let(:new_string) { "Hello \n World" } @@ -75,5 +75,22 @@ RSpec.describe Gitlab::Diff::CharDiff do 'd' ) end + + context 'when changes involve newlines' do + let(:old_string) { "Hello\nWorld" } + let(:new_string) { "Hello World\n" } + + it 'replaces newlines with ↵' do + subject.generate_diff + + expect(subject.to_html).to eq( + 'Hello' \ + "↵\n" \ + ' ' \ + 'World' \ + "↵\n" + ) + end + end end end diff --git a/spec/lib/gitlab/instrumentation/redis_cluster_router_spec.rb b/spec/lib/gitlab/instrumentation/redis_cluster_router_spec.rb new file mode 100644 index 00000000000..8cdb178128a --- /dev/null +++ b/spec/lib/gitlab/instrumentation/redis_cluster_router_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'rspec-parameterized' + +RSpec.describe Gitlab::Instrumentation::RedisClusterRouter, feature_category: :redis do + describe '#send_command', if: ::Gitlab::Redis::Cache.params[:nodes] do + before do + Gitlab::Redis::Cache.with do |c| + allow(c._client.instance_variable_get(:@router)) + .to receive(:assign_node) + .and_raise(::RedisClient::Cluster::NodeMightBeDown) + end + end + + it 'tracks exception' do + expect(Gitlab::ErrorTracking) + .to receive(:log_exception).with( + instance_of(::RedisClient::Cluster::NodeMightBeDown), node_keys: anything, slots_map: anything + ) + + expect(Gitlab::Instrumentation::Redis::Cache) + .to receive(:instance_count_exception).with(instance_of(::RedisClient::Cluster::NodeMightBeDown)) + + expect do + Gitlab::Redis::Cache.with { |c| c.decr('test') } + end.to raise_error(Redis::Cluster::NodeMightBeDown) # redis-rb gem intercepts and replaces error + end + end + + describe '.format_slotmap' do + it 'handles empty slot array' do + expect(described_class.format_slotmap([])).to eq({}) + end + + it 'handles incomplete slot array' do + input = ['localhost:1', 'localhost:1', nil, 'localhost:2'] + expect(described_class.format_slotmap(input)).to eq({ 'localhost:1' => '0-1', 'localhost:2' => '3-3' }) + end + + it 'handles complete slot array' do + input = (['localhost:1'] * 5000) + (['localhost:2'] * 5000) + (['localhost:3'] * 5000) + (['localhost:1'] * 5000) + expect(described_class.format_slotmap(input)) + .to eq({ + 'localhost:1' => '0-4999,15000-16383', 'localhost:2' => '5000-9999', 'localhost:3' => '10000-14999' + }) + end + end + + describe '.compact_array' do + using RSpec::Parameterized::TableSyntax + + where(:input, :output) do + [1, 2, 3, 4, 5, 6, 7] | "1-7" # contiguous + [1, 2, 3, 5, 6, 7, 9, 10] | "1-3,5-7,9-10" # non-contiguous + [1, 2, 3, 4, 5, 7] | "1-5,7-7" # contigious with 1 ending slot + [] | "" # empty + [1, 1, 1, 1] | "1-1" # homogenuous array + end + + with_them do + it do + expect(described_class.compact_array(input)).to eq(output) + end + end + end +end diff --git a/spec/support/factory_default.rb b/spec/support/factory_default.rb index 31af022f6c0..a13355392f8 100644 --- a/spec/support/factory_default.rb +++ b/spec/support/factory_default.rb @@ -10,7 +10,7 @@ module Gitlab end end -TestProf::FactoryDefault::DefaultSyntax.prepend Gitlab::FreezeFactoryDefault +TestProf::FactoryDefault::FactoryBotPatch::SyntaxExt.prepend Gitlab::FreezeFactoryDefault RSpec.configure do |config| config.after do |ex|