root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2024-06-02 12:13:44 +00:00
parent 7a60e4ef2b
commit 3afe77ca78
4 changed files with 269 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
qa/qa/support/formatters/test_metrics_formatter.rb
View File

@ -7,6 +7,9 @@ module QA
module Formatters
class TestMetricsFormatter < RSpec::Core::Formatters::BaseFormatter
include Support::InfluxdbTools
include Support::GcsTools
include Support::Repeater
include Support::Retrier
CUSTOM_METRICS_KEY = :custom_test_metrics
@ -44,51 +47,111 @@ module QA
def execution_data(examples = nil)
@execution_metrics ||= examples.filter_map { |example| test_stats(example) }
end
alias_method :parse_execution_data, :execution_data
# Push test execution metrics to influxdb
# Upload test execution metrics
#
# @return [void]
def push_test_metrics
return log(:debug, "Metrics export not enabled, skipping test metrics export") unless export_metrics?
return log(:info, "Metrics export not enabled, skipping test metrics export") unless export_metrics?
push_test_metrics_to_influxdb
push_test_metrics_to_gcs
end
# Push test execution metrics to InfluxDB
#
# @return [void]
def push_test_metrics_to_influxdb
write_api.write(data: execution_data)
log(:debug, "Pushed #{execution_data.length} test execution entries to influxdb")
log(:info, "Pushed #{execution_data.length} test execution entries to influxdb")
rescue StandardError => e
log(:error, "Failed to push test execution metrics to influxdb, error: #{e}")
end
# Push test execution metrics to GCS
#
# @return [void]
def push_test_metrics_to_gcs
retry_on_exception(sleep_interval: 30, message: 'Failed to push test metrics to GCS') do
gcs_client.put_object(gcs_bucket, metrics_file_name(prefix: 'test'), execution_data.to_json,
force: true, content_type: 'application/json')
log(:info, "Pushed #{execution_data.length} test execution entries to GCS")
end
end
# Push resource fabrication metrics to influxdb
#
# @return [void]
def push_fabrication_metrics
return log(:debug, "Metrics export not enabled, skipping fabrication metrics export") unless export_metrics?
return log(:info, "Metrics export not enabled, skipping fabrication metrics export") unless export_metrics?
data = Tools::TestResourceDataProcessor.resources.flat_map do |resource, values|
values.map { |v| fabrication_stats(resource: resource, **v) }
end
return if data.empty?
push_fabrication_metrics_influxdb(data)
push_fabrication_metrics_gcs(data)
end
# Push resource fabrication metrics to GCS
#
# @param [Hash] data fabrication data hash
# @return [void]
def push_fabrication_metrics_gcs(data)
retry_on_exception(sleep_interval: 30, message: 'Failed to push resource fabrication metrics to GCS') do
gcs_client.put_object(gcs_bucket,
metrics_file_name(prefix: 'fabrication'), data.to_json, force: true, content_type: 'application/json')
log(:info, "Pushed #{data.length} resource fabrication entries to GCS")
end
end
# Push resource fabrication metrics to InfluxDB
#
# @param [Hash] data fabrication data hash
# @return [void]
def push_fabrication_metrics_influxdb(data)
write_api.write(data: data)
log(:debug, "Pushed #{data.length} resource fabrication entries to influxdb")
log(:info, "Pushed #{data.length} resource fabrication entries to influxdb")
rescue StandardError => e
log(:error, "Failed to push fabrication metrics to influxdb, error: #{e}")
end
# Get GCS Bucket Name or raise error if missing
#
# @return [String]
def gcs_bucket
@gcs_bucket ||= ENV['QA_METRICS_GCS_BUCKET_NAME'] ||
raise('Missing QA_METRICS_GCS_BUCKET_NAME env variable')
end
# Save metrics in json file
#
# @return [void]
def save_test_metrics
return log(:debug, "Saving test metrics json not enabled, skipping") unless save_metrics_json?
return log(:info, "Saving test metrics json not enabled, skipping") unless save_metrics_json?
file = "tmp/test-metrics-#{env('CI_JOB_NAME_SLUG') || 'local'}" \
"#{retry_failed_specs? ? "-retry-#{rspec_retried?}" : ''}.json"
file = File.join('tmp', metrics_file_name(prefix: 'test'))
File.write(file, execution_data.to_json) && log(:debug, "Saved test metrics to #{file}")
rescue StandardError => e
log(:error, "Failed to save test execution metrics, error: #{e}")
end
# Construct file name for metrics
#
# @param [Hash] prefix of filename
# @return [void]
def metrics_file_name(prefix:)
"#{prefix}-metrics-#{env('CI_JOB_NAME_SLUG') || 'local'}" \
"#{retry_failed_specs? ? "-retry-#{rspec_retried?}" : ''}.json"
end
# Transform example to influxdb compatible metrics data
# https://github.com/influxdata/influxdb-client-ruby#data-format
#
@ -154,6 +217,7 @@ module QA
pipeline_id: env('CI_PIPELINE_ID'),
job_id: env('CI_JOB_ID'),
merge_request_iid: merge_request_iid,
failure_issue: example.metadata[:quarantine] ? example.metadata[:quarantine][:issue] : nil,
failure_exception: example.execution_result.exception.to_s.delete("\n"),
**custom_metrics_fields(example.metadata)
}.compact

46
qa/qa/tools/ci/test_metrics.rb
View File

@ -8,6 +8,9 @@ module QA
class TestMetrics
include Helpers
include Support::InfluxdbTools
include Support::GcsTools
include Support::Repeater
include Support::Retrier
def initialize(metrics_file_glob)
@metrics_file_glob = metrics_file_glob
@ -23,10 +26,8 @@ module QA
def export
return logger.warn("No files matched pattern '#{metrics_file_glob}'") if metrics_files.empty?
logger.info("Exporting #{metrics_data.size} entries to influxdb")
influx_client
.create_write_api(write_options: write_options)
.write(data: metrics_data, bucket: INFLUX_MAIN_TEST_METRICS_BUCKET)
push_test_metrics_to_influxdb
push_test_metrics_to_gcs
end
private
@ -59,6 +60,43 @@ module QA
.flat_map { |file| JSON.parse(File.read(file), symbolize_names: true) }
.map { |entry| entry.merge(time: entry[:time].to_time) }
end
# Get GCS Bucket Name or raise error if missing
#
# @return [String]
def gcs_bucket
@gcs_bucket ||= ENV['QA_METRICS_GCS_BUCKET_NAME'] ||
raise('Missing QA_METRICS_GCS_BUCKET_NAME env variable')
end
# Push test execution metrics to GCS
#
# @return [void]
def push_test_metrics_to_gcs
date = Time.now
gcs_file_name = "main_test-stats_#{date.strftime('%Y-%m-%d')}-#{date.to_i}.json"
retry_on_exception(sleep_interval: 30, message: 'Failed to push test metrics to GCS') do
gcs_client.put_object(gcs_bucket, gcs_file_name, metrics_data.to_json,
force: true, content_type: 'application/json')
logger.info("Pushed #{metrics_data.length} test execution entries to GCS")
end
end
# Push test execution metrics to InfluxDB
#
# @return [void]
def push_test_metrics_to_influxdb
logger.info("Exporting #{metrics_data.size} entries to influxdb")
influx_client
.create_write_api(write_options: write_options)
.write(data: metrics_data, bucket: INFLUX_MAIN_TEST_METRICS_BUCKET)
rescue StandardError => e
logger.error("Failed to push test execution metrics to influxdb, error: #{e}")
end
end
end
end

158
qa/spec/support/formatters/test_metrics_formatter_spec.rb
View File

@ -11,6 +11,11 @@ describe QA::Support::Formatters::TestMetricsFormatter do
let(:url) { 'http://influxdb.net' }
let(:token) { 'token' }
let(:metrics_gcs_project_id) { 'metrics-gcs-project' }
let(:metrics_gcs_creds) { 'metrics-gcs-creds' }
let(:metrics_gcs_bucket_name) { 'metrics-gcs-bucket' }
let(:gcs_client_options) { { force: true, content_type: 'application/json' } }
let(:gcs_client) { double("Fog::Storage::GoogleJSON::Real", put_object: nil) } # rubocop:disable RSpec/VerifiedDoubles -- instance_double complains put_object is not implemented but it is
let(:ci_timestamp) { '2021-02-23T20:58:41Z' }
let(:ci_job_name) { 'test-job 1/5' }
let(:ci_job_url) { 'url' }
@ -21,6 +26,7 @@ describe QA::Support::Formatters::TestMetricsFormatter do
let(:smoke) { 'false' }
let(:blocking) { 'false' }
let(:quarantined) { 'false' }
let(:failure_issue) { '' }
let(:influx_client) { instance_double('InfluxDB2::Client', create_write_api: influx_write_api) }
let(:influx_write_api) { instance_double('InfluxDB2::WriteApi', write: nil) }
let(:file_path) { "./qa/specs/features/1_manage/subfolder/some_spec.rb" }
@ -99,6 +105,10 @@ describe QA::Support::Formatters::TestMetricsFormatter do
before do
allow(::Gitlab::QA::Runtime::Env).to receive(:retry_failed_specs?).and_return(retry_failed_specs)
allow(InfluxDB2::Client).to receive(:new).with(url, token, **influx_client_args) { influx_client }
allow(Fog::Storage::Google).to receive(:new)
.with(google_project: metrics_gcs_project_id,
google_json_key_string: metrics_gcs_creds)
.and_return(gcs_client)
allow(QA::Tools::TestResourceDataProcessor).to receive(:resources) { fabrication_resources }
allow_any_instance_of(RSpec::Core::Example::ExecutionResult).to receive(:run_time).and_return(0) # rubocop:disable RSpec/AnyInstanceOf -- simplifies mocking runtime
end
@ -123,13 +133,42 @@ describe QA::Support::Formatters::TestMetricsFormatter do
end
end
context 'with influxdb variables configured' do
context 'without GCS variables configured' do
it 'skips export without gcs creds' do
stub_env('QA_METRICS_GCS_CREDS', nil)
run_spec
expect(gcs_client).not_to have_received(:put_object)
end
it 'skips export without gcs project id' do
stub_env('QA_METRICS_GCS_PROJECT_ID', nil)
run_spec
expect(gcs_client).not_to have_received(:put_object)
end
it 'skips export without gcs bucket name' do
stub_env('QA_METRICS_GCS_BUCKET_NAME', nil)
run_spec
expect(gcs_client).not_to have_received(:put_object)
end
end
context 'with influxdb and GCS variables configured' do
let(:spec_name) { 'exports data' }
let(:run_type) { ci_job_name.gsub(%r{ \d{1,2}/\d{1,2}}, '') }
before do
stub_env('QA_INFLUXDB_URL', url)
stub_env('QA_INFLUXDB_TOKEN', token)
stub_env('QA_METRICS_GCS_PROJECT_ID', metrics_gcs_project_id)
stub_env('QA_METRICS_GCS_CREDS', metrics_gcs_creds)
stub_env('QA_METRICS_GCS_BUCKET_NAME', metrics_gcs_bucket_name)
stub_env('CI_PIPELINE_CREATED_AT', ci_timestamp)
stub_env('CI_JOB_URL', ci_job_url)
stub_env('CI_JOB_NAME', ci_job_name)
@ -147,25 +186,30 @@ describe QA::Support::Formatters::TestMetricsFormatter do
context 'with blocking spec' do
let(:blocking) { 'true' }
it 'exports data to influxdb with correct blocking tag' do
it 'exports data with correct blocking tag', :aggregate_failures do
run_spec do
it('spec', :blocking, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {}
end
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
anything, [data].to_json, **gcs_client_options)
end
end
context 'with product group tag' do
it 'exports data to influxdb with correct blocking tag' do
let(:expected_data) { [data.tap { |d| d[:tags][:product_group] = :import }] }
it 'exports data with correct product group tag' do
run_spec do
it('spec', product_group: :import, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {}
end
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(
data: [data.tap { |d| d[:tags][:product_group] = :import }]
data: expected_data
)
end
end
@ -173,56 +217,102 @@ describe QA::Support::Formatters::TestMetricsFormatter do
context 'with smoke spec' do
let(:smoke) { 'true' }
it 'exports data to influxdb with correct smoke tag' do
it 'exports data with correct blocking tag', :aggregate_failures do
run_spec do
it('spec', :smoke, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {}
end
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
anything, [data].to_json, **gcs_client_options)
end
end
context 'with quarantined spec' do
let(:quarantined) { 'true' }
let(:status) { :pending }
let(:expected_data) do
data.tap do |d|
d[:fields] = {
id: './spec/support/formatters/test_metrics_formatter_spec.rb[1:1]',
run_time: 0,
api_fabrication: api_fabrication * 1000,
ui_fabrication: ui_fabrication * 1000,
total_fabrication: (api_fabrication + ui_fabrication) * 1000,
job_url: ci_job_url,
pipeline_url: ci_pipeline_url,
pipeline_id: ci_pipeline_id,
job_id: ci_job_id,
failure_issue: 'https://example.com/issue/1234',
failure_exception: ''
}
end
end
it 'exports data to influxdb with correct quarantine tag' do
it 'exports data with correct quarantine tag', :aggregate_failures do
run_spec do
it(
'spec',
:quarantine,
quarantine: {
type: :stale,
issue: 'https://example.com/issue/1234'
},
skip: 'quarantined',
testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234'
) {}
end
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
expect(influx_write_api).to have_received(:write).with(data: [expected_data])
expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
anything, [expected_data].to_json, **gcs_client_options)
end
end
context 'with context quarantined spec' do
let(:quarantined) { 'false' }
let(:expected_data) do
data.tap do |d|
d[:fields] = {
id: './spec/support/formatters/test_metrics_formatter_spec.rb[1:1]',
run_time: 0,
api_fabrication: api_fabrication * 1000,
ui_fabrication: ui_fabrication * 1000,
total_fabrication: (api_fabrication + ui_fabrication) * 1000,
job_url: ci_job_url,
pipeline_url: ci_pipeline_url,
pipeline_id: ci_pipeline_id,
job_id: ci_job_id,
failure_issue: 'https://example.com/issue/1234',
failure_exception: ''
}
end
end
it 'exports data to influxdb with correct quarantine tag' do
it 'exports data with correct quarantine tag', :aggregate_failures do
run_spec do
it(
'spec',
quarantine: { only: { job: 'praefect' } },
quarantine: { only: { job: 'praefect' }, issue: 'https://example.com/issue/1234' },
testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234'
) {}
end
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
expect(influx_write_api).to have_received(:write).with(data: [expected_data])
expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
anything, [expected_data].to_json, **gcs_client_options)
end
end
context 'with skipped spec' do
let(:status) { :pending }
it 'exports data with pending status' do
it 'exports data with pending status', :aggregate_failures do
run_spec do
it(
'spec',
@ -232,20 +322,27 @@ describe QA::Support::Formatters::TestMetricsFormatter do
end
expect(influx_write_api).to have_received(:write).with(data: [data])
expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
anything, [data].to_json, **gcs_client_options)
end
end
context 'with failed spec' do
let(:status) { :failed }
let(:expected_data) { data.tap { |d| d[:tags][:exception_class] = "RuntimeError" } }
it 'saves exception class' do
it 'saves exception class', :aggregate_failures do
run_spec(passed: false) do
it('spec', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') { raise }
end
expect(influx_write_api).to have_received(:write).with(
data: [data.tap { |d| d[:tags][:exception_class] = "RuntimeError" }]
data: [expected_data]
)
expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
anything, [expected_data].to_json, **gcs_client_options)
end
end
@ -253,12 +350,14 @@ describe QA::Support::Formatters::TestMetricsFormatter do
let(:retry_failed_specs) { true }
context 'with initial run' do
it 'skips failed spec' do
it 'skips failed spec', :aggregate_failures do
run_spec(passed: false) do
it('spec', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') { raise }
end
expect(influx_write_api).to have_received(:write).with(data: [])
expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
anything, [].to_json, **gcs_client_options)
end
end
@ -269,12 +368,14 @@ describe QA::Support::Formatters::TestMetricsFormatter do
stub_env('QA_RSPEC_RETRIED', 'true')
end
it 'sets test as flaky' do
it 'sets test as flaky', :aggregate_failures do
run_spec do
it('spec', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {}
end
expect(influx_write_api).to have_received(:write).with(data: [data])
expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
anything, [data].to_json, **gcs_client_options)
end
end
end
@ -287,16 +388,19 @@ describe QA::Support::Formatters::TestMetricsFormatter do
stub_env('QA_RUN_TYPE', nil)
end
it 'exports data to influxdb with correct run type' do
it 'exports data with correct run type', :aggregate_failures do
run_spec
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
anything, [data].to_json, **gcs_client_options)
end
end
context 'with additional custom metrics' do
it 'exports data to influxdb with additional metrics' do
it 'exports data additional metrics', :aggregate_failures do
run_spec do
it(
'spec',
@ -313,6 +417,9 @@ describe QA::Support::Formatters::TestMetricsFormatter do
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [custom_data])
expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
anything, [custom_data].to_json, **gcs_client_options)
end
end
@ -320,7 +427,7 @@ describe QA::Support::Formatters::TestMetricsFormatter do
let(:api_fabrication) { 4 }
let(:ui_fabrication) { 10 }
it 'exports data to influxdb with fabrication times' do
it 'exports data with fabrication times', :aggregate_failures do
run_spec do
# Main logic tracks fabrication time in thread local variable and injects it as metadata from
# global after hook defined in main spec_helper.
@ -336,6 +443,9 @@ describe QA::Support::Formatters::TestMetricsFormatter do
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
anything, [data].to_json, **gcs_client_options)
end
end
@ -343,12 +453,15 @@ describe QA::Support::Formatters::TestMetricsFormatter do
let(:file_path) { './qa/specs/features/shared_examples/merge_with_code_owner_shared_examples.rb' }
let(:rerun_file_path) { './qa/specs/features/3_create/subfolder/another_spec.rb' }
it 'exports data to influxdb with correct filename' do
it 'exports data to influxdb with correct filename', :aggregate_failures do
run_spec
data[:tags][:file_path] = '/3_create/subfolder/another_spec.rb'
data[:tags][:stage] = 'create'
expect(influx_write_api).to have_received(:write).with(data: [data])
expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
anything, [data].to_json, **gcs_client_options)
end
end
@ -390,10 +503,13 @@ describe QA::Support::Formatters::TestMetricsFormatter do
freeze_time { example.run }
end
it 'exports fabrication stats data to influxdb' do
it 'exports fabrication stats data to influxdb and GCS', :aggregate_failures do
run_spec
expect(influx_write_api).to have_received(:write).with(data: [fabrication_data])
expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
anything, [fabrication_data].to_json, **gcs_client_options)
end
end

19
qa/spec/tools/ci/test_metrics_spec.rb
View File

@ -5,6 +5,8 @@ RSpec.describe QA::Tools::Ci::TestMetrics do
let(:influx_client) { instance_double("InfluxDB2::Client", create_write_api: influx_write_api) }
let(:influx_write_api) { instance_double("InfluxDB2::WriteApi", write: nil) }
let(:gcs_client_options) { { force: true, content_type: 'application/json' } }
let(:gcs_client) { double("Fog::Storage::GoogleJSON::Real", put_object: nil) } # rubocop:disable RSpec/VerifiedDoubles -- Class has `put_object` method but is not getting verified
let(:logger) { instance_double("Logger", info: true, warn: true) }
let(:glob) { "metrics_glob/*.json" }
@ -12,6 +14,10 @@ RSpec.describe QA::Tools::Ci::TestMetrics do
let(:timestamp) { "2022-11-11 07:54:11 +0000" }
let(:metrics_json) { metrics_data.to_json }
let(:metrics_gcs_project_id) { 'metrics-gcs-project' }
let(:metrics_gcs_creds) { 'metrics-gcs-creds' }
let(:metrics_gcs_bucket_name) { 'metrics-gcs-bucket' }
let(:metrics_data) do
[
{
@ -25,19 +31,30 @@ RSpec.describe QA::Tools::Ci::TestMetrics do
before do
allow(InfluxDB2::Client).to receive(:new) { influx_client }
allow(Fog::Storage::Google).to receive(:new)
.with(google_project: metrics_gcs_project_id,
google_json_key_string: metrics_gcs_creds)
.and_return(gcs_client)
allow(Gitlab::QA::TestLogger).to receive(:logger) { logger }
allow(Dir).to receive(:glob).with(glob) { paths }
allow(File).to receive(:read).with(paths.first) { metrics_json }
stub_env('QA_INFLUXDB_URL', "test")
stub_env('QA_INFLUXDB_TOKEN', "test")
stub_env('QA_METRICS_GCS_PROJECT_ID', metrics_gcs_project_id)
stub_env('QA_METRICS_GCS_CREDS', metrics_gcs_creds)
stub_env('QA_METRICS_GCS_BUCKET_NAME', metrics_gcs_bucket_name)
end
context "with metrics files present" do
it "exports saved metrics to influxdb" do
it "exports saved metrics to influxdb and GCS", :aggregate_failures do
described_class.export(glob)
expect(influx_write_api).to have_received(:write).with(data: metrics_data, bucket: "e2e-test-stats-main")
expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
anything, metrics_data.to_json, **gcs_client_options)
end
end