root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2024-10-01 03:14:26 +00:00
parent 69895fd56f
commit cf2bc7ae20
18 changed files with 126 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitlab/ci/test-on-gdk/main.gitlab-ci.yml
View File

@ -83,7 +83,7 @@ include:
GITLAB_QA_ADMIN_ACCESS_TOKEN: $QA_ADMIN_ACCESS_TOKEN
FF_NETWORK_PER_BUILD: "true"
RSPEC_LAST_RUN_RESULTS_FILE: "$CI_PROJECT_DIR/qa/tmp/examples.txt"
COVERBAND_ENABLED: "$COVERBAND_ENABLED"
COVERBAND_ENABLED: "$COVERBAND_ENABLED" # explicitly define variable so it is passed in to gdk service container
before_script:
- echo "SUITE_RAN=true" > "$QA_SUITE_STATUS_ENV_FILE"
- echo -e "\e[0Ksection_start:`date +%s`:install_gems[collapsed=true]\r\e[0KInstall gems"

1
.rubocop_todo/layout/line_length.yml
View File

@ -2015,7 +2015,6 @@ Layout/LineLength:
- 'ee/spec/services/requirements_management/export_csv_service_spec.rb'
- 'ee/spec/services/resource_events/change_weight_service_spec.rb'
- 'ee/spec/services/search/global_service_spec.rb'
- 'ee/spec/services/search/project_service_spec.rb'
- 'ee/spec/services/search/snippet_service_spec.rb'
- 'ee/spec/services/security/ingestion/finding_map_collection_spec.rb'
- 'ee/spec/services/security/ingestion/ingest_report_service_spec.rb'

1
.rubocop_todo/rspec/context_wording.yml
View File

@ -738,7 +738,6 @@ RSpec/ContextWording:
- 'ee/spec/services/resource_access_tokens/create_service_spec.rb'
- 'ee/spec/services/resource_access_tokens/revoke_service_spec.rb'
- 'ee/spec/services/search/global_service_spec.rb'
- 'ee/spec/services/search/project_service_spec.rb'
- 'ee/spec/services/search/snippet_service_spec.rb'
- 'ee/spec/services/security/ingestion/tasks/ingest_vulnerabilities/create_spec.rb'
- 'ee/spec/services/security/ingestion/tasks/update_vulnerability_uuids_spec.rb'

9
app/assets/stylesheets/page_bundles/_ide_monaco_overrides.scss
View File

@ -5,10 +5,6 @@
flex-direction: column;
justify-content: center;
.vertical-center {
min-height: auto;
}
.monaco-editor .lines-content .cigr {
display: none;
}
@ -117,11 +113,6 @@
border-right: 1px solid $line-removed-dark;
}
}
.margin-view-overlays .insert-sign,
.margin-view-overlays .delete-sign {
opacity: 0.4;
}
}
}

15
config/coverband.rb Normal file
View File

@ -0,0 +1,15 @@
# frozen_string_literal: true
# This file is loaded from 'initializers/coverband.rb' if coverband is enabled
Coverband.configure do |config|
config.store = Coverband::Adapters::RedisStore.new(Gitlab::Redis::SharedState.redis)
config.background_reporting_sleep_seconds = 1
config.reporting_wiggle = 0 # Since this is not run in production disable wiggle and report every second.
config.ignore += %w[spec/.* lib/tasks/.*
config/application.rb config/boot.rb config/initializers/.* db/post_migrate/.*
config/puma.rb bin/.* config/environments/.* db/migrate/.*]
config.verbose = false # this spams logfile a lot, set to true for debugging locally
config.logger = Gitlab::AppLogger.primary_logger
end

16
config/initializers/coverband.rb
View File

@ -3,17 +3,7 @@
# Configuration used by coverband gem when "COVERBAND_ENABLED" is set to "true"
return unless Gitlab::Utils.to_boolean(ENV['COVERBAND_ENABLED'], default: false)
# Set coverband config file name so it is not started with default configuration
ENV["COVERBAND_CONFIG"] = Rails.root.join("config/coverband.rb").to_s
require 'coverband'
Coverband.configure do |config|
config.store = Coverband::Adapters::RedisStore.new(Gitlab::Redis::SharedState.redis)
config.background_reporting_sleep_seconds = 1
config.reporting_wiggle = nil # Since this is not run in production disable wiggle and report every second.
config.ignore += %w[spec/.* lib/tasks/.*
config/application.rb config/boot.rb config/initializers/.* db/post_migrate/.*
config/puma.rb bin/.* config/environments/.* db/migrate/.*]
config.verbose = true
config.csp_policy = true
config.logger = Gitlab::AppLogger
end

1
data/deprecations/17-0-deprecate-support-for-OWASP-2017.yml
View File

@ -11,5 +11,6 @@
body: | # (required) Don't change this line.
Grouping the vulnerability report by OWASP top 10 2017 is deprecated, replaced by grouping by OWASP top 10 2021.
In the future we will support the most recent version of OWASP top 10 for grouping on the vulnerability report.
Along with this change we are also deprecating and removing the 2017 GraphQL API enums which the feature uses. Additional details are included in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/488433).
tiers: [Gold, Ultimate]
documentation_url: https://docs.gitlab.com/ee/user/application_security/vulnerability_report/#group-vulnerabilities

1
doc/update/deprecations.md
View File

@ -1081,6 +1081,7 @@ See the [GitLab 17.5 changes](https://docs.gitlab.com/ee/update/versions/gitlab_
Grouping the vulnerability report by OWASP top 10 2017 is deprecated, replaced by grouping by OWASP top 10 2021.
In the future we will support the most recent version of OWASP top 10 for grouping on the vulnerability report.
Along with this change we are also deprecating and removing the 2017 GraphQL API enums which the feature uses. Additional details are included in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/488433).
</div>

10
lib/api/internal/coverage.rb
View File

@ -22,12 +22,18 @@ module API
end
get do
coverage = ::Coverband.configuration.store.coverage
coverage&.keys
# Fetch only runtime coverage data which is tracked during E2E spec execution
# skip hash check due to hash mismatch on some environments which results in empty coverage data
::Coverband.configuration.store.coverage(::Coverband::RUNTIME_TYPE, skip_hash_check: true).keys
end
delete do
::Coverband.configuration.store.clear!
status 200
{
message: "Cleared source code paths coverage mapping"
}
end
end
end

38
lib/gitlab/patch/old_redis_cache_store.rb
View File

@ -9,7 +9,9 @@ module Gitlab
return super unless enable_rails_cache_pipeline_patch?
return super unless use_patched_mget?
patched_read_multi_mget(*names) # rubocop:disable Style/ArgumentsForwarding -- Overridden patch
::Gitlab::Redis::ClusterUtil.batch_entries(names) do |batched_names|
super(*batched_names)
end.reduce(&:merge)
end
# `delete_multi_entries` in Rails runs a multi-key `del` command
@ -17,37 +19,9 @@ module Gitlab
def delete_multi_entries(entries, **options)
return super unless enable_rails_cache_pipeline_patch?
redis.with do |conn|
::Gitlab::Redis::ClusterUtil.batch_del(entries, conn)
end
end
# Copied from https://github.com/rails/rails/blob/v6.1.6.1/activesupport/lib/active_support/cache/redis_cache_store.rb
# re-implements `read_multi_mget` using a pipeline of `get`s rather than an `mget`
#
def patched_read_multi_mget(*names)
options = names.extract_options!
options = merged_options(options)
return {} if names == []
raw = options&.fetch(:raw, false)
keys = names.map { |name| normalize_key(name, options) }
values = failsafe(:patched_read_multi_mget, returning: {}) do
redis.with do |c|
::Gitlab::Redis::ClusterUtil.batch_get(keys, c)
end
end
names.zip(values).each_with_object({}) do |(name, value), results|
if value # rubocop:disable Style/Next -- Overridden patch
entry = deserialize_entry(value, raw: raw)
unless entry.nil? || entry.expired? || entry.mismatched?(normalize_version(name, options))
results[name] = entry.value
end
end
end
::Gitlab::Redis::ClusterUtil.batch_entries(entries) do |batched_names|
super(batched_names)
end.sum
end
private

38
lib/gitlab/patch/redis_cache_store.rb
View File

@ -5,11 +5,13 @@ module Gitlab
module RedisCacheStore
# We will try keep patched code explicit and matching the original signature in
# https://github.com/rails/rails/blob/v7.1.3.4/activesupport/lib/active_support/cache/redis_cache_store.rb#L324
def read_multi_entries(...)
def read_multi_entries(names, **options) # rubocop:disable Style/ArgumentsForwarding -- Overridden patch
return super unless enable_rails_cache_pipeline_patch?
return super unless use_patched_mget?
patched_read_multi_entries(...)
::Gitlab::Redis::ClusterUtil.batch_entries(names) do |batched_names|
super(batched_names, **options)
end.reduce(&:merge)
end
# `delete_multi_entries` in Rails runs a multi-key `del` command
@ -17,9 +19,9 @@ module Gitlab
def delete_multi_entries(entries, **options)
return super unless enable_rails_cache_pipeline_patch?
redis.with do |conn|
::Gitlab::Redis::ClusterUtil.batch_del(entries, conn)
end
::Gitlab::Redis::ClusterUtil.batch_entries(entries) do |batched_names|
super(batched_names)
end.sum
end
# `pipeline_entries` is used by Rails for multi-key writes
@ -32,32 +34,6 @@ module Gitlab
end
end
# Copied from https://github.com/rails/rails/blob/v7.1.3.4/activesupport/lib/active_support/cache/redis_cache_store.rb#L324
# re-implements `read_multi_entries` using a pipeline of `get`s rather than an `mget`
def patched_read_multi_entries(names, **options)
options = merged_options(options)
return {} if names == []
raw = options&.fetch(:raw, false)
keys = names.map { |name| normalize_key(name, options) }
values = failsafe(:patched_read_multi_entries, returning: {}) do
redis.with do |c|
::Gitlab::Redis::ClusterUtil.batch_get(keys, c)
end
end
names.zip(values).each_with_object({}) do |(name, value), results|
next unless value
entry = deserialize_entry(value, raw: raw)
unless entry.nil? || entry.expired? || entry.mismatched?(normalize_version(name, options))
results[name] = entry.value
end
end
end
private
def enable_rails_cache_pipeline_patch?

6
lib/gitlab/redis/cluster_util.rb
View File

@ -36,6 +36,12 @@ module Gitlab
end.flatten
end
def batch_entries(entries)
entries.each_slice(pipeline_batch_size).flat_map do |subset|
yield subset
end
end
def batch(entries, redis)
entries.each_slice(pipeline_batch_size).flat_map do |subset|
redis.pipelined do |pipeline|

2
qa/gdk/Dockerfile.gdk
View File

@ -1,4 +1,4 @@
ARG GDK_SHA=a5a1c84f9a0625231b09b03c8dadcc2322a3e2bb
ARG GDK_SHA=705133b39da1b5eb9b4d75fc7f73b5992512e32b
# Use tag prefix when running on 'stable' branch to make sure 'protected' image is used which is not deleted by registry cleanup
ARG GDK_BASE_TAG_PREFIX

58
qa/qa/support/formatters/coverband_formatter.rb
View File

@ -9,13 +9,13 @@ module QA
class CoverbandFormatter < ::RSpec::Core::Formatters::BaseFormatter
include Support::API
COVERAGE_API_PATH = '/api/v4/internal/coverage'
def initialize(output)
super
@test_mapping = Hash.new { |hsh, key| hsh[key] = [] }
@logger = Runtime::Logger.logger
@headers_access_token = { "PRIVATE-TOKEN" => Runtime::Env.admin_personal_access_token }
@cov_api_endpoint = "#{Runtime::Scenario.gitlab_address}/api/v4/internal/coverage"
end
::RSpec::Core::Formatters.register(
@ -30,45 +30,49 @@ module QA
# @param [RSpec::Core::Notifications::ExamplesNotification] notification
# @return [void]
def stop(_notification)
logger.info("Saving test coverage mapping json file")
save_test_mapping
end
# Example start event
def example_started(_example_notification)
QA::Support::Retrier.retry_until(max_attempts: 5, sleep_interval: 1, message: "Retry clear coverage") do
resp = delete("#{Runtime::Scenario.gitlab_address}#{COVERAGE_API_PATH}",
headers: headers_access_token)
def example_started(example_notification)
return if example_notification.example.metadata[:skip]
logger.error("Failed to clear coverage, code: #{resp.code}, body: #{resp.body}") if resp.code != 200
response = nil
QA::Support::Retrier.retry_until(max_attempts: 5, sleep_interval: 1) do
response = delete(cov_api_endpoint, headers: headers_access_token)
next true if response.code == 200
resp.code == 200
logger.debug("Failed to clear coverage, code: #{response.code}, body: #{response.body}")
false
end
logger.debug("Cleared coverage data before example starts")
rescue StandardError => e
logger.error("Failed to clear coverage. Exception trace: #{e}")
logger.info("Cleared coverage data")
rescue StandardError
logger.error("Failed to clear coverage, code: #{response.code}, body: #{response.body}")
end
# Example finish event
def example_finished(example_notification)
cov_resp = nil
QA::Support::Retrier.retry_until(max_attempts: 10, sleep_interval: 2, message: "Retry fetch coverage") do
cov_resp = get("#{Runtime::Scenario.gitlab_address}/api/v4/internal/coverage",
headers: headers_access_token)
return if example_notification.example.metadata[:skip] || example_failed?(example_notification)
if cov_resp.code != 200
logger.error("Fetching coverage data failed, code: #{cov_resp.code}, body: #{cov_resp.body}")
response = nil
QA::Support::Retrier.retry_until(max_attempts: 10, sleep_interval: 2) do
response = get(cov_api_endpoint, headers: headers_access_token)
coverage = JSON.parse(response.body)
next true if response.code == 200 && coverage.any?
if response.code != 200
logger.debug("Fetching coverage data failed, code: #{response.code}, body: #{response.body}")
end
cov_resp.code == 200 && !JSON.parse(cov_resp.body).empty?
logger.debug("Fetching coverage data failed, no coverage data available") if coverage.empty?
false
end
example_path = example_notification.example.metadata[:location]
test_mapping[example_path] = JSON.parse(cov_resp.body) unless example_failed?(example_notification)
logger.debug("Coverage paths were stored in mapping hash")
rescue StandardError => e
logger.error("Failed to fetch coverage mapping. Trace: #{e}")
test_mapping[example_path] = JSON.parse(response.body)
logger.info("Fetched coverage data")
rescue StandardError
logger.error("Failed to fetch coverage data, code: #{response.code}, body: #{response.body}")
end
def example_failed?(example_notification)
@ -83,14 +87,14 @@ module QA
# To write two different files in case of failed specs being retried
File.write(file, test_mapping.to_json)
logger.debug("Saved test code paths mapping to #{file}")
logger.info("Saved test coverage mapping data to #{file}")
rescue StandardError => e
logger.error("Failed to save test code paths mapping, error: #{e}")
logger.error("Failed to save test coverage mapping data, error: #{e}")
end
private
attr_reader :test_mapping, :logger, :headers_access_token
attr_reader :test_mapping, :logger, :headers_access_token, :cov_api_endpoint
end
end
end

93
qa/spec/support/formatters/coverband_formatter_spec.rb
View File

@ -9,10 +9,8 @@ describe QA::Support::Formatters::CoverbandFormatter do
instance_double(RSpec::Core::Notifications::ExampleNotification, example: rspec_example)
end
# rubocop:disable RSpec/VerifiedDoubles -- Custom object
let(:rspec_example) do
double(
instance_double(
RSpec::Core::Example,
file_path: 'create_issue_spec.rb',
execution_result: instance_double(RSpec::Core::Example::ExecutionResult, status: status),
@ -24,22 +22,14 @@ describe QA::Support::Formatters::CoverbandFormatter do
)
end
# rubocop:enable RSpec/VerifiedDoubles
let(:gitlab_address) { 'http://gitlab.test.com' }
let(:api_path) { "#{gitlab_address}/api/v4/internal/coverage" }
let(:status) { :failed }
let(:token_header) { { "PRIVATE-TOKEN" => 'token' } }
let(:api_response) do
instance_double(
::RestClient::Response
)
end
let(:non_empty_response) do
'{"test mapping":1}'
end
let(:logger) { instance_double(ActiveSupport::Logger, debug: true, error: true, info: true) }
let(:api_response) { instance_double(::RestClient::Response, code: code, body: body) }
let(:code) { 200 }
let(:body) { '[]' }
let(:mapping) do
{ "./qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb:5": { "test mapping": 1 } }
@ -47,108 +37,103 @@ describe QA::Support::Formatters::CoverbandFormatter do
before do
stub_env('GITLAB_QA_ADMIN_ACCESS_TOKEN', 'token')
allow(QA::Runtime::Logger).to receive(:logger).and_return(logger)
allow(QA::Runtime::Scenario).to receive(:gitlab_address).and_return(gitlab_address)
allow(::RestClient::Request).to receive(:execute).and_return(api_response)
end
context 'with example_started' do
before do
allow(QA::Runtime::Scenario).to receive(:gitlab_address).and_return(gitlab_address)
allow(api_response).to receive(:body).and_return({})
allow(::RestClient::Request).to receive(:execute).and_return(api_response)
end
context 'when success response' do
before do
allow(api_response).to receive(:code).and_return(200)
end
it 'logs coverage cleared', :aggregate_failures do
expect(QA::Runtime::Logger.logger).to receive(:debug).with("Cleared coverage data before example starts").once
formatter.example_started(rspec_example_notification)
expect(logger).to have_received(:info).with("Cleared coverage data").once
end
end
context 'with failure response' do
let(:code) { 401 }
before do
allow(api_response).to receive(:code).and_return(401)
allow(QA::Support::Retrier).to receive(:retry_until).and_wrap_original do |method|
method.call(max_attempts: 1)
allow(QA::Support::Retrier).to receive(:retry_until).and_wrap_original do |method, *_args, &block|
method.call(max_attempts: 1, &block)
end
end
it 'logs error message' do
expect(QA::Runtime::Logger.logger).to receive(:error).with(/Failed to clear coverage.*/).once
formatter.example_started(rspec_example_notification)
expect(logger).to have_received(:error).with("Failed to clear coverage, code: #{code}, body: #{body}").once
end
end
end
context 'when example finished' do
before do
allow(QA::Runtime::Scenario).to receive(:gitlab_address).and_return(gitlab_address)
allow(::RestClient::Request).to receive(:execute).and_return(api_response)
end
context 'with success response and non empty coverage' do
let(:status) { :passed }
before do
allow(api_response).to receive(:code).and_return(200)
allow(api_response).to receive(:body).and_return(non_empty_response)
end
let(:body) { '{"test mapping":1}' }
it 'logs success message and does not log any errors' do
expect(QA::Runtime::Logger.logger).not_to receive(:error)
expect(QA::Runtime::Logger.logger).to receive(:debug).with("Coverage paths were stored in mapping hash").once
formatter.example_finished(rspec_example_notification)
expect(logger).not_to have_received(:error)
expect(logger).to have_received(:info).with("Fetched coverage data").once
end
end
context 'with failure response' do
let(:status) { :passed }
let(:code) { 401 }
before do
allow(api_response).to receive(:code).and_return(401)
allow(api_response).to receive(:body).and_return({})
allow(QA::Support::Retrier).to receive(:retry_until).and_wrap_original do |method|
method.call(max_attempts: 1)
allow(QA::Support::Retrier).to receive(:retry_until).and_wrap_original do |method, *_args, &block|
method.call(max_attempts: 1, &block)
end
end
it 'logs error message' do
expect(QA::Runtime::Logger.logger).to receive(:error).with(/Failed to fetch coverage mapping.*/).once
formatter.example_finished(rspec_example_notification)
expect(logger).to have_received(:error)
.with("Failed to fetch coverage data, code: #{code}, body: #{body}")
.once
end
end
end
context 'when save_test_mapping is called' do
let(:file_name_pattern) { "test-code-paths-mapping-job-name" }
before do
stub_env('CI_JOB_NAME_SLUG', 'job-name')
end
let(:file_name_pattern) { "test-code-paths-mapping-job-name" }
context 'with mapping data present' do
before do
allow(formatter).to receive(:test_mapping).and_return(mapping)
allow(::File).to receive(:write)
end
it 'writes to file' do
expect(::File).to receive(:write).with(/#{file_name_pattern}/, mapping.to_json).once
expect(QA::Runtime::Logger.logger).to receive(:debug).with(/Saved test code paths mapping to.*/).once
formatter.save_test_mapping
expect(::File).to have_received(:write).with(/#{file_name_pattern}/, mapping.to_json).once
expect(logger).to have_received(:info).with(/Saved test coverage mapping data to \S+\.json/).once
end
end
context 'when writing to file throws an error' do
before do
allow(::File).to receive(:write).and_raise(StandardError)
allow(::File).to receive(:write).and_raise("some error")
end
it 'raises an error' do
expect(QA::Runtime::Logger.logger).to receive(:error)
.with(/Failed to save test code paths mapping, error:.*/).once
formatter.save_test_mapping
expect(logger).to have_received(:error)
.with("Failed to save test coverage mapping data, error: some error")
.once
end
end
end

8
spec/lib/gitlab/patch/redis_cache_store_spec.rb
View File

@ -48,11 +48,7 @@ RSpec.describe Gitlab::Patch::RedisCacheStore, :use_clean_rails_redis_caching, f
if normal_cluster || multistore_cluster
times = (input_size.to_f / chunk_size).ceil
expect(redis).to receive(:pipelined).exactly(times).times.and_call_original
expect_next_instances_of(::Redis::PipelinedConnection, times) do |p|
expect(p).to receive(:get).at_most(chunk_size).times
end
expect(redis).to receive(:mget).exactly(times).times.and_call_original
else
expect(redis).to receive(:mget).and_call_original
end
@ -111,7 +107,7 @@ RSpec.describe Gitlab::Patch::RedisCacheStore, :use_clean_rails_redis_caching, f
# no expectation on number of times as it could vary depending on cluster size
# if the Redis is a Redis Cluster
if Gitlab::Redis::ClusterUtil.cluster?(redis)
expect(redis).to receive(:pipelined).at_least(2).and_call_original
expect(redis).to receive(:del).at_least(2).and_call_original
else
expect(redis).to receive(:del).and_call_original
end

6
spec/requests/api/internal/coverage_spec.rb
View File

@ -36,8 +36,12 @@ RSpec.describe API::Internal::Coverage, feature_category: :code_testing do
before do
stub_const('Coverband', Class.new)
allow(Coverband).to receive_message_chain(:configuration, :store, :coverage).and_return(coverage_hash)
stub_const('Coverband::RUNTIME_TYPE', :runtime)
allow(Coverband).to receive_message_chain(:configuration, :store, :clear!).and_return({})
allow(Coverband).to receive_message_chain(:configuration, :store, :coverage)
.with(:runtime, skip_hash_check: true)
.and_return(coverage_hash)
end
it 'GET returns 200', :aggregate_failures do

1
spec/support/rspec_order_todo.yml
View File

@ -2408,7 +2408,6 @@
- './ee/spec/services/resource_access_tokens/revoke_service_spec.rb'
- './ee/spec/services/resource_events/change_weight_service_spec.rb'
- './ee/spec/services/search/global_service_spec.rb'
- './ee/spec/services/search/project_service_spec.rb'
- './ee/spec/services/search_service_spec.rb'
- './ee/spec/services/search/snippet_service_spec.rb'
- './ee/spec/services/security/ingestion/finding_map_collection_spec.rb'