root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2024-07-26 06:11:22 +00:00
parent 15c19cd0c3
commit 51655b8280
20 changed files with 268 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.rubocop_todo/gitlab/bounded_contexts.yml
View File

@ -3021,7 +3021,6 @@ Gitlab/BoundedContexts:
- 'ee/app/models/geo_node_namespace_link.rb'
- 'ee/app/models/geo_node_status.rb'
- 'ee/app/models/gitlab_subscription.rb'
- 'ee/app/models/gitlab_subscription_history.rb'
- 'ee/app/models/group_deletion_schedule.rb'
- 'ee/app/models/group_merge_request_approval_setting.rb'
- 'ee/app/models/group_wiki.rb'

1
.rubocop_todo/gitlab/namespaced_class.yml
View File

@ -879,7 +879,6 @@ Gitlab/NamespacedClass:
- 'ee/app/models/geo_node_status.rb'
- 'ee/app/models/gitlab/seat_link_data.rb'
- 'ee/app/models/gitlab_subscription.rb'
- 'ee/app/models/gitlab_subscription_history.rb'
- 'ee/app/models/group_deletion_schedule.rb'
- 'ee/app/models/group_merge_request_approval_setting.rb'
- 'ee/app/models/group_wiki.rb'

3
app/assets/javascripts/releases/components/app_index.vue
View File

@ -21,8 +21,7 @@ export default {
i18n,
links: {
alertInfoMessageLink: helpPagePath('ci/yaml/index.html', { anchor: 'release' }),
// eslint-disable-next-line local-rules/require-valid-help-page-path
alertInfoPublishLink: helpPagePath('ci/components/index', { anchor: 'release-a-component' }),
alertInfoPublishLink: helpPagePath('ci/components/index', { anchor: 'publish-a-new-release' }),
},
components: {
GlAlert,

9
app/finders/bulk_imports/imports_finder.rb
View File

@ -9,7 +9,8 @@ module BulkImports
def execute
imports = filter_by_status(user.bulk_imports)
sort(imports)
imports = sort(imports)
include_configuration(imports)
end
private
@ -27,5 +28,11 @@ module BulkImports
imports.order_by_created_at(@params[:sort])
end
def include_configuration(imports)
return imports unless @params[:include_configuration]
imports.with_configuration
end
end
end

5
app/models/bulk_import.rb
View File

@ -21,6 +21,7 @@ class BulkImport < ApplicationRecord
scope :stale, -> { where('updated_at < ?', 24.hours.ago).where(status: [0, 1]) }
scope :order_by_updated_at_and_id, ->(direction) { order(updated_at: direction, id: :asc) }
scope :order_by_created_at, ->(direction) { order(created_at: direction) }
scope :with_configuration, -> { includes(:configuration) }
state_machine :status, initial: :created do
state :created, value: 0
@ -119,4 +120,8 @@ class BulkImport < ApplicationRecord
def parent_group_entity
entities.group_entity.where(parent: nil).first
end
def source_url
configuration&.url
end
end

2
db/docs/gitlab_subscription_histories.yml
View File

@ -1,7 +1,7 @@
---
table_name: gitlab_subscription_histories
classes:
- GitlabSubscriptionHistory
- GitlabSubscriptions::SubscriptionHistory
feature_categories:
- subscription_management
description: History log for the gitlab_subscriptions table

4
doc/api/bulk_imports.md
View File

@ -74,6 +74,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token_for_destination_
"id": 1,
"status": "created",
"source_type": "gitlab",
"source_url": "https://gitlab.example.com",
"created_at": "2021-06-18T09:45:55.358Z",
"updated_at": "2021-06-18T09:46:27.003Z",
"has_failures": false
@ -110,6 +111,7 @@ curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab
"id": 1,
"status": "finished",
"source_type": "gitlab",
"source_url": "https://gitlab.example.com",
"created_at": "2021-06-18T09:45:55.358Z",
"updated_at": "2021-06-18T09:46:27.003Z",
"has_failures": false
@ -118,6 +120,7 @@ curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab
"id": 2,
"status": "started",
"source_type": "gitlab",
"source_url": "https://gitlab.example.com",
"created_at": "2021-06-18T09:47:36.581Z",
"updated_at": "2021-06-18T09:47:58.286Z",
"has_failures": false
@ -231,6 +234,7 @@ curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab
"id": 1,
"status": "finished",
"source_type": "gitlab",
"source_url": "https://gitlab.example.com",
"created_at": "2021-06-18T09:45:55.358Z",
"updated_at": "2021-06-18T09:46:27.003Z"
}

2
lib/api/bulk_imports.rb
View File

@ -11,7 +11,7 @@ module API
def bulk_imports
@bulk_imports ||= ::BulkImports::ImportsFinder.new(
user: current_user,
params: params
params: params.merge(include_configuration: true)
).execute
end

1
lib/api/entities/bulk_import.rb
View File

@ -8,6 +8,7 @@ module API
type: 'string', example: 'finished', values: %w[created started finished timeout failed]
}
expose :source_type, documentation: { type: 'string', example: 'gitlab' }
expose :source_url, documentation: { type: 'string', example: 'https://source.gitlab.com/' }
expose :created_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' }
expose :updated_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' }
expose :has_failures, documentation: { type: 'boolean', example: false }

5
lib/tasks/gitlab/backup.rake
View File

@ -75,7 +75,10 @@ module Tasks
yield
ensure
backup_progress.puts(
"#{Time.current} #{Rainbow('-- Deleting backup and restore PID file ...').blue} #{Rainbow('done').green}"
"#{Time.current} " +
'-- Deleting backup and restore PID file at ['.color(:blue) +
PID_FILE.to_s + '] ... '.color(:blue) +
'done'.color(:green)
)
File.delete(PID_FILE)
end

4
qa/qa/runtime/env.rb
View File

@ -94,6 +94,10 @@ module QA
enabled?(ENV['COVERBAND_ENABLED'], default: false)
end
def selective_execution_improved_enabled?
enabled?(ENV['SELECTIVE_EXECUTION_IMPROVED'], default: false)
end
def schedule_type
ENV['SCHEDULE_TYPE']
end

51
qa/qa/tools/ci/export_code_paths_mapping.rb → qa/qa/tools/ci/code_paths_mapping.rb
View File

@ -5,24 +5,21 @@ require "fog/google"
module QA
module Tools
module Ci
class ExportCodePathsMapping
class CodePathsMapping
include Helpers
PROJECT = "gitlab-qa-resources"
BUCKET = "code-path-mappings"
def self.export(mapping_files_glob)
new(mapping_files_glob).export
end
def initialize(mapping_files_glob)
@mapping_files_glob = mapping_files_glob
new.export(mapping_files_glob)
end
# Export code path mappings to GCP
#
# @param [String] mapping_files_glob - glob pattern for mapping files
# @return [void]
def export
def export(mapping_files_glob)
mapping_files = Dir.glob(mapping_files_glob)
return logger.warn("No files matched pattern, skipping coverage mapping upload") if mapping_files.empty?
@ -38,9 +35,24 @@ module QA
upload_to_gcs(file, mapping_data)
end
private
# Import code path mappings from GCP
#
# @param [String] branch - branch name
# @param [String] run_type - run type
# @return [Hash]
def import(branch, run_type)
filename = code_paths_mapping_file("#{branch}/#{run_type}")
attr_reader :mapping_files_glob
logger.info("The mapping file fetched in import: #{filename}")
file = client.get_object(BUCKET, filename)
JSON.parse(file[:body])
rescue StandardError => e
logger.error("Failed to download code paths mapping from GCS. Error: #{e}")
logger.error("Backtrace: #{e.backtrace}")
nil # Ensure it returns nil in case of GCS errors
end
private
def upload_to_gcs(file_name, mapping_data)
client.put_object(BUCKET, file_name, JSON.pretty_generate(mapping_data))
@ -68,6 +80,27 @@ module QA
{ google_json_key_string: json_key }
end
# Code paths mapping file from GCS
#
# Get most up to date mapping file based on pipeline type
# @return [String]
def code_paths_mapping_file(prefix)
paginated_list(client.list_objects(BUCKET, prefix: prefix)).last&.name
end
# Paginated list of items
#
# @param [Google::Apis::StorageV1::Objects] list
# @return [Array]
def paginated_list(list)
return [] if list.items.nil?
return list.items if list.next_page_token.nil?
paginated_list(
client.list_objects(BUCKET, prefix: list.prefixes.first, page_token: list.next_page_token)
) + list.items
end
end
end
end

36
qa/qa/tools/ci/qa_changes.rb
View File

@ -38,7 +38,20 @@ module QA
.join(" ")
end
qa_spec_directories_for_devops_stage&.join(" ") if non_qa_changes? && mr_labels.any?
fetch_list_of_specs_or_directories
end
# Fetches list of E2E specs from code paths mapping OR list of spec directories from stage labels
#
# @return [String]
def fetch_list_of_specs_or_directories
if QA::Runtime::Env.selective_execution_improved_enabled? && non_qa_changes?
tests = selective_tests_from_code_paths_mapping
logger.info("Selected tests from mapping: '#{tests}'")
tests.nil? || tests.empty? ? nil : tests.join(" ")
elsif non_qa_changes? && mr_labels.any?
qa_spec_directories_for_devops_stage&.join(" ")
end
end
# Qa framework changes
@ -87,6 +100,9 @@ module QA
# @return [Array]
attr_reader :mr_labels
# @return [Array]
attr_reader :selected_e2e_tests
# @return [Hash<String, Array<String>>]
attr_reader :additional_group_spec_list
@ -151,6 +167,24 @@ module QA
@changed_files ||= mr_diff.map { |change| change[:path] }
end
# Selective E2E tests based on code paths mapping
#
# @return [array]
def selective_tests_from_code_paths_mapping
clean_map = code_paths_map&.each_with_object({}) do |(test_filename, code_mappings), hsh|
name = test_filename.gsub("./", "").split(":").first
hsh[name] = hsh.key?(name) ? (code_mappings + hsh[name]).uniq : code_mappings
end
clean_map&.select { |_test, mappings| changed_files.any? { |file| mappings.include?("./#{file}") } }&.keys
end
# Get the mapping hash from GCP storage
#
# @return [Hash]
def code_paths_map
@code_paths_map ||= QA::Tools::Ci::CodePathsMapping.new.import("master", "e2e-test-on-gdk")
end
# Devops stage specs
#
# @param [Array<String>] devops_stages

55
qa/spec/tools/ci/export_code_paths_mapping_spec.rb → qa/spec/tools/ci/code_paths_mapping_spec.rb
View File

@ -1,7 +1,10 @@
# frozen_string_literal: true
RSpec.describe QA::Tools::Ci::ExportCodePathsMapping do
require 'active_support/testing/time_helpers'
RSpec.describe QA::Tools::Ci::CodePathsMapping do
include QA::Support::Helpers::StubEnv
include ActiveSupport::Testing::TimeHelpers
let(:glob) { "test_code_paths/*.json" }
let(:file_paths) { ["/test_code/test_code_path_mappings.json"] }
@ -57,4 +60,54 @@ RSpec.describe QA::Tools::Ci::ExportCodePathsMapping do
described_class.export(glob)
end
end
context "with import" do
subject(:code_paths_mapping) { described_class.new }
let(:branch) { "master" }
let(:run_type) { "e2e-test-on-gdk" }
let(:file_path_2) { "#{branch}/#{run_type}/test-code-paths-mapping-merged-pipeline-2.json" }
let(:google_api_object_1) do
instance_double('Google::Apis::StorageV1::Object',
name: "test_path", bucket: "code-path-mappings")
end
let(:google_api_object_2) do
instance_double('Google::Apis::StorageV1::Object',
name: file_path_2, bucket: "code-path-mappings")
end
let(:mapping_files_list) do
instance_double('Google::Apis::StorageV1::Objects', items:
[google_api_object_1, google_api_object_2], next_page_token: nil)
end
context "when mapping file present for pipeline type" do
let(:response_from_gcs) { { name: "file_name", body: "{}" } }
before do
allow(gcs_client).to receive(:list_objects).and_return(mapping_files_list)
allow(gcs_client).to receive(:get_object).with(gcs_bucket_name, String).and_return(response_from_gcs)
end
it 'calls get_object with correct mapping file path' do
expect(gcs_client).to receive(:get_object).with(gcs_bucket_name, file_path_2)
code_paths_mapping.import(branch, run_type)
end
end
context "when mapping file cannot be retrieved" do
before do
allow(code_paths_mapping).to receive(:code_paths_mapping_file).and_return(nil)
allow(gcs_client).to receive(:get_object).with(gcs_bucket_name, nil).and_raise(ArgumentError)
end
it 'logs the error and does not raise an exception' do
expect(logger).to receive(:error).with("Failed to download code paths mapping from GCS. Error: ArgumentError")
expect(logger).to receive(:error).with(/Backtrace: \[.*/)
code_paths_mapping.import(branch, run_type)
end
end
end
end

62
qa/spec/tools/ci/qa_changes_spec.rb
View File

@ -1,6 +1,9 @@
# frozen_string_literal: true
require "fog/google"
RSpec.describe QA::Tools::Ci::QaChanges do
include QA::Support::Helpers::StubEnv
subject(:qa_changes) { described_class.new(mr_diff, mr_labels, additional_group_spec_list) }
let(:mr_labels) { [] }
@ -8,6 +11,7 @@ RSpec.describe QA::Tools::Ci::QaChanges do
before do
allow(File).to receive(:directory?).and_return(false)
stub_env('SELECTIVE_EXECUTION_IMPROVED', false)
end
context "with spec only changes" do
@ -85,6 +89,64 @@ RSpec.describe QA::Tools::Ci::QaChanges do
end
end
context "with SELECTIVE_EXECUTION_IMPROVED enabled" do
let(:code_paths_mapping_data) do
{
"./qa/specs/features/test_spec.rb:23": %w[./lib/model.rb ./lib/second.rb],
"./qa/specs/features/test_spec_2.rb:11": ['./app/controller.rb']
}.stringify_keys
end
let(:selected_specs) { "qa/specs/features/test_spec.rb" }
let(:gcs_project_id) { 'gitlab-qa-resources' }
let(:gcs_creds) { 'gcs-creds' }
let(:gcs_bucket_name) { 'metrics-gcs-bucket' }
let(:gcs_client) { double("Fog::Storage::GoogleJSON::Real", put_object: nil) } # rubocop:disable RSpec/VerifiedDoubles -- instance_double complains put_object is not implemented but it is
let(:code_paths_mapping) do
instance_double(QA::Tools::Ci::CodePathsMapping, import: code_paths_mapping_data)
end
before do
stub_env('SELECTIVE_EXECUTION_IMPROVED', true)
stub_env('QA_CODE_PATH_MAPPINGS_GCS_CREDENTIALS', gcs_creds)
allow(Fog::Storage::Google).to receive(:new)
.with(google_project: gcs_project_id,
google_json_key_string: gcs_creds)
.and_return(gcs_client)
allow(QA::Tools::Ci::CodePathsMapping).to receive(:new).and_return(code_paths_mapping)
end
describe '#qa_tests' do
context 'when there is a match from code paths mapping' do
let(:mr_diff) { [{ path: 'lib/model.rb' }] }
it "returns specific specs" do
expect(qa_changes.qa_tests.split(" ")).to include(selected_specs)
end
end
context 'when there is no match from code paths mapping' do
let(:mr_diff) { [{ path: 'lib/new.rb' }] }
it "returns nil" do
expect(qa_changes.qa_tests).to be_nil
end
end
context 'when code paths mapping import returns nil' do
let(:mr_diff) { [{ path: 'lib/model.rb' }] }
let(:code_paths_mapping) do
instance_double(QA::Tools::Ci::CodePathsMapping, import: nil)
end
it "does not throw an error" do
expect(qa_changes.qa_tests).to be_nil
end
end
end
end
context "when configured to run tests from other stages" do
let(:additional_group_spec_list) do
{

2
qa/tasks/ci.rake
View File

@ -88,6 +88,6 @@ namespace :ci do
task :export_code_paths_mapping, [:glob] do |_, args|
raise("Code paths mapping JSON glob pattern is required") unless args[:glob]
QA::Tools::Ci::ExportCodePathsMapping.export(args[:glob])
QA::Tools::Ci::CodePathsMapping.export(args[:glob])
end
end

14
spec/finders/bulk_imports/imports_finder_spec.rb
View File

@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe BulkImports::ImportsFinder do
RSpec.describe BulkImports::ImportsFinder, feature_category: :importers do
let_it_be(:user) { create(:user) }
let_it_be(:started_import) { create(:bulk_import, :started, user: user) }
let_it_be(:finished_import) { create(:bulk_import, :finished, user: user) }
@ -31,7 +31,7 @@ RSpec.describe BulkImports::ImportsFinder do
end
end
context 'when order is specifed' do
context 'when order is specified' do
subject { described_class.new(user: user, params: { sort: order }) }
context 'when order is specified as asc' do
@ -50,5 +50,15 @@ RSpec.describe BulkImports::ImportsFinder do
end
end
end
context 'when configuration is included' do
it 'preloads configuration association' do
imports = described_class
.new(user: user, params: { include_configuration: true })
.execute
expect(imports.first.association_cached?(:configuration)).to be(true)
end
end
end
end

9
spec/lib/api/entities/bulk_import_spec.rb
View File

@ -3,7 +3,7 @@
require 'spec_helper'
RSpec.describe API::Entities::BulkImport, feature_category: :importers do
let_it_be(:import) { create(:bulk_import) }
let_it_be(:import) { create(:bulk_import, :with_configuration) }
subject { described_class.new(import).as_json }
@ -12,9 +12,16 @@ RSpec.describe API::Entities::BulkImport, feature_category: :importers do
:id,
:status,
:source_type,
:source_url,
:created_at,
:updated_at,
:has_failures
)
end
it 'exposes source url via configuration' do
expected_url = import.configuration.url
expect(subject[:source_url]).to eq(expected_url)
end
end

24
spec/models/bulk_import_spec.rb
View File

@ -44,6 +44,14 @@ RSpec.describe BulkImport, type: :model, feature_category: :importers do
])
end
end
describe '.with_configuration' do
it 'includes configuration association' do
imports = described_class.with_configuration
expect(imports.first.association_cached?(:configuration)).to be(true)
end
end
end
describe '.all_human_statuses' do
@ -206,4 +214,20 @@ RSpec.describe BulkImport, type: :model, feature_category: :importers do
expect(import.parent_group_entity).to eq(root_node)
end
end
describe '#source_url' do
it 'returns migration source url via configuration' do
import = create(:bulk_import, :with_configuration)
expect(import.source_url).to eq(import.configuration.url)
end
context 'when configuration is missing' do
it 'returns nil' do
import = create(:bulk_import)
expect(import.source_url).to be_nil
end
end
end
end

2
spec/tasks/gitlab/backup_rake_spec.rb
View File

@ -60,7 +60,7 @@ RSpec.describe 'gitlab:backup namespace rake tasks', :reestablished_active_recor
describe 'lock parallel backups' do
let(:progress) { $stdout }
let(:delete_message) { /-- Deleting backup and restore PID file/ }
let(:delete_message) { /-- Deleting backup and restore PID file at/ }
let(:pid_file) do
File.open(backup_restore_pid_path, File::RDWR | File::CREAT)
end