Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2023-11-29 21:09:59 +00:00
parent 2d239d2421
commit b5eff9f970
21 changed files with 270 additions and 51 deletions

View File

@ -5,7 +5,7 @@ workflow:
name: $PIPELINE_NAME
include:
- component: "gitlab.com/gitlab-org/quality/pipeline-common/allure-report@7.10.0"
- component: "gitlab.com/gitlab-org/quality/pipeline-common/allure-report@7.13.1"
inputs:
job_name: "e2e-test-report"
job_stage: "report"

View File

@ -197,9 +197,9 @@ gem 'seed-fu', '~> 2.3.7' # rubocop:todo Gemfile/MissingFeatureCategory
gem 'elasticsearch-model', '~> 7.2' # rubocop:todo Gemfile/MissingFeatureCategory
gem 'elasticsearch-rails', '~> 7.2', require: 'elasticsearch/rails/instrumentation' # rubocop:todo Gemfile/MissingFeatureCategory
gem 'elasticsearch-api', '7.13.3' # rubocop:todo Gemfile/MissingFeatureCategory
gem 'aws-sdk-core', '~> 3.188.0' # rubocop:todo Gemfile/MissingFeatureCategory
gem 'aws-sdk-core', '~> 3.189.0' # rubocop:todo Gemfile/MissingFeatureCategory
gem 'aws-sdk-cloudformation', '~> 1' # rubocop:todo Gemfile/MissingFeatureCategory
gem 'aws-sdk-s3', '~> 1.139.0' # rubocop:todo Gemfile/MissingFeatureCategory
gem 'aws-sdk-s3', '~> 1.141.0' # rubocop:todo Gemfile/MissingFeatureCategory
gem 'faraday_middleware-aws-sigv4', '~>0.3.0' # rubocop:todo Gemfile/MissingFeatureCategory
gem 'typhoeus', '~> 1.4.0' # Used with Elasticsearch to support http keep-alive connections # rubocop:todo Gemfile/MissingFeatureCategory

View File

@ -34,13 +34,13 @@
{"name":"autoprefixer-rails","version":"10.2.5.1","platform":"ruby","checksum":"3711d67f1112361c7628847ac192d8aa6f3b8abe47527aee8a69dc8985e798ee"},
{"name":"awesome_print","version":"1.9.2","platform":"ruby","checksum":"e99b32b704acff16d768b3468680793ced40bfdc4537eb07e06a4be11133786e"},
{"name":"awrence","version":"1.2.1","platform":"ruby","checksum":"dd1d214c12a91f449d1ef81d7ee3babc2816944e450752e7522c65521872483e"},
{"name":"aws-eventstream","version":"1.2.0","platform":"ruby","checksum":"ffa53482c92880b001ff2fb06919b9bb82fd847cbb0fa244985d2ebb6dd0d1df"},
{"name":"aws-eventstream","version":"1.3.0","platform":"ruby","checksum":"f1434cc03ab2248756eb02cfa45e900e59a061d7fbdc4a9fd82a5dd23d796d3f"},
{"name":"aws-partitions","version":"1.761.0","platform":"ruby","checksum":"291e444e1edfc92c5521a6dbdd1236ccc3f122b3520163b2be6ec5b6ef350ef2"},
{"name":"aws-sdk-cloudformation","version":"1.41.0","platform":"ruby","checksum":"31e47539719734413671edf9b1a31f8673fbf9688549f50c41affabbcb1c6b26"},
{"name":"aws-sdk-core","version":"3.188.0","platform":"ruby","checksum":"67b785c60466e5f459e250a38f55f5170ab6723db8740157bb6b06a7888d6337"},
{"name":"aws-sdk-core","version":"3.189.0","platform":"ruby","checksum":"a2b5d55c2f3827c8e453a228b011ac5c0074a09c301e289774151225114c7ecf"},
{"name":"aws-sdk-kms","version":"1.64.0","platform":"ruby","checksum":"40de596c95047bfc6e1aacea24f3df6241aa716b6f7ce08ac4c5f7e3120395ad"},
{"name":"aws-sdk-s3","version":"1.139.0","platform":"ruby","checksum":"a7cf63382f93c5790687371daa6723101db8976aa6a9a4ba7f2305a2103fc87b"},
{"name":"aws-sigv4","version":"1.6.0","platform":"ruby","checksum":"ca9e6a15cd424f1f32b524b9760995331459bc22e67d3daad4fcf0c0084b087d"},
{"name":"aws-sdk-s3","version":"1.141.0","platform":"ruby","checksum":"cadb88497af6736e86a4a1fc8eb42333fb27ae85901686334252c50862bdd02e"},
{"name":"aws-sigv4","version":"1.8.0","platform":"ruby","checksum":"84dd99768b91b93b63d1d8e53ee837cfd06ab402812772a7899a78f9f9117cbc"},
{"name":"axe-core-api","version":"4.6.0","platform":"ruby","checksum":"1b0ddec3353f108dc10363baf2282f43a5ff7f13d4e25f99071294e78f8a6c62"},
{"name":"axe-core-rspec","version":"4.6.0","platform":"ruby","checksum":"11c25bc9dd388c137ba4e5e63d64d20092bf22c884d8ffc829a22acfbacd747f"},
{"name":"axiom-types","version":"0.1.1","platform":"ruby","checksum":"c1ff113f3de516fa195b2db7e0a9a95fd1b08475a502ff660d04507a09980383"},

View File

@ -286,24 +286,24 @@ GEM
execjs (> 0)
awesome_print (1.9.2)
awrence (1.2.1)
aws-eventstream (1.2.0)
aws-eventstream (1.3.0)
aws-partitions (1.761.0)
aws-sdk-cloudformation (1.41.0)
aws-sdk-core (~> 3, >= 3.99.0)
aws-sigv4 (~> 1.1)
aws-sdk-core (3.188.0)
aws-eventstream (~> 1, >= 1.0.2)
aws-sdk-core (3.189.0)
aws-eventstream (~> 1, >= 1.3.0)
aws-partitions (~> 1, >= 1.651.0)
aws-sigv4 (~> 1.5)
aws-sigv4 (~> 1.8)
jmespath (~> 1, >= 1.6.1)
aws-sdk-kms (1.64.0)
aws-sdk-core (~> 3, >= 3.165.0)
aws-sigv4 (~> 1.1)
aws-sdk-s3 (1.139.0)
aws-sdk-core (~> 3, >= 3.188.0)
aws-sdk-s3 (1.141.0)
aws-sdk-core (~> 3, >= 3.189.0)
aws-sdk-kms (~> 1)
aws-sigv4 (~> 1.6)
aws-sigv4 (1.6.0)
aws-sigv4 (~> 1.8)
aws-sigv4 (1.8.0)
aws-eventstream (~> 1, >= 1.0.2)
axe-core-api (4.6.0)
dumb_delegator
@ -1792,8 +1792,8 @@ DEPENDENCIES
autoprefixer-rails (= 10.2.5.1)
awesome_print
aws-sdk-cloudformation (~> 1)
aws-sdk-core (~> 3.188.0)
aws-sdk-s3 (~> 1.139.0)
aws-sdk-core (~> 3.189.0)
aws-sdk-s3 (~> 1.141.0)
axe-core-rspec
babosa (~> 2.0)
base32 (~> 0.3.0)

View File

@ -5,7 +5,7 @@
module DisablesSti
extend ActiveSupport::Concern
SKIP_STI_CHECK = Gitlab.dev_or_test_env? &&
SKIP_STI_CHECK = !Gitlab.dev_or_test_env? ||
Gitlab::Utils.to_boolean(ENV['SKIP_STI_CHECK'], default: false)
included do

View File

@ -10,6 +10,8 @@ class PipelineScheduleWorker # rubocop:disable Scalability/IdempotentWorker
LOCK_RETRY = 3
LOCK_TTL = 5.minutes
DELAY = 7.seconds
BATCH_SIZE = 1000
feature_category :continuous_integration
worker_resource_boundary :cpu
@ -20,12 +22,8 @@ class PipelineScheduleWorker # rubocop:disable Scalability/IdempotentWorker
.select(:id, :owner_id, :project_id) # Minimize the selected columns
.runnable_schedules
.preloaded
.find_in_batches do |schedules|
RunPipelineScheduleWorker.bulk_perform_async_with_contexts(
schedules,
arguments_proc: ->(schedule) { [schedule.id, schedule.owner_id, { scheduling: true }] },
context_proc: ->(schedule) { { project: schedule.project, user: schedule.owner } }
)
.find_in_batches(batch_size: BATCH_SIZE).with_index do |schedules, index| # rubocop: disable CodeReuse/ActiveRecord -- activates because of batch_size
enqueue_run_pipeline_schedule_worker(schedules, index + 1)
end
end
end
@ -42,4 +40,21 @@ class PipelineScheduleWorker # rubocop:disable Scalability/IdempotentWorker
retries: LOCK_RETRY
}
end
def enqueue_run_pipeline_schedule_worker(schedules, index)
if ::Feature.enabled?(:run_pipeline_schedule_worker_with_delay)
RunPipelineScheduleWorker.bulk_perform_in_with_contexts(
index * DELAY,
schedules,
arguments_proc: ->(schedule) { [schedule.id, schedule.owner_id, { scheduling: true }] },
context_proc: ->(schedule) { { project: schedule.project, user: schedule.owner } }
)
else
RunPipelineScheduleWorker.bulk_perform_async_with_contexts(
schedules,
arguments_proc: ->(schedule) { [schedule.id, schedule.owner_id, { scheduling: true }] },
context_proc: ->(schedule) { { project: schedule.project, user: schedule.owner } }
)
end
end
end

View File

@ -10,7 +10,7 @@ class RunPipelineScheduleWorker # rubocop:disable Scalability/IdempotentWorker
queue_namespace :pipeline_creation
feature_category :continuous_integration
deduplicate :until_executed
deduplicate :until_executed, including_scheduled: true
idempotent!
def perform(schedule_id, user_id, options = {})

View File

@ -0,0 +1,8 @@
---
name: run_pipeline_schedule_worker_with_delay
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/136992
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/431892
milestone: '16.7'
type: development
group: group::pipeline execution
default_enabled: false

View File

@ -226,7 +226,7 @@ The turnaround time to process configuration change requests is [documented in t
To enable the Inbound Private Link:
1. In the body of your [support ticket](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=4414917877650), include the IAM Principal for the AWS user or role in your own AWS Organization that's establishing the VPC endpoint in your AWS account. GitLab Dedicated uses this IAM Principal for access-control. This IAM principal is the only one able to set up an endpoint to the service.
1. Open a [support ticket](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=4414917877650). In the body of your support ticket, include the IAM principal for the AWS user or role in your AWS organization that's establishing the VPC endpoint in your AWS account. The IAM principal must be an [IAM role principal](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_principal.html#principal-roles) or [IAM user principal](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_principal.html#principal-users). GitLab Dedicated uses this IAM Principal for access-control. This IAM principal is the only one able to set up an endpoint to the service.
1. After your IAM Principal has been allowlisted, GitLab [creates the Endpoint Service](https://docs.aws.amazon.com/vpc/latest/privatelink/create-endpoint-service.html) and communicates the `Service Endpoint Name` on the support ticket. The service name is generated by AWS upon creation of the service endpoint.
- GitLab handles the domain verification for the Private DNS name, so that DNS resolution of the tenant instance domain name in your VPC resolves to the PrivateLink endpoint.
- GitLab makes the Endpoint Service available in the Availability Zones you specified during the initial onboarding. If you did not specify any Availability Zones, GitLab randomly selects the Availability Zones IDs.

View File

@ -505,7 +505,7 @@ is stored on the machine where GitLab Runner is installed. The location also dep
| Runner executor | Default path of the cache |
| ---------------------- | ------------------------- |
| [Shell](https://docs.gitlab.com/runner/executors/shell.html) | Locally, under the `gitlab-runner` user's home directory: `/home/gitlab-runner/cache/<user>/<project>/<cache-key>/cache.zip`. |
| [Docker](https://docs.gitlab.com/runner/executors/docker.html) | Locally, under [Docker volumes](https://docs.gitlab.com/runner/executors/docker.html#the-builds-and-cache-storage): `/var/lib/docker/volumes/<volume-id>/_data/<user>/<project>/<cache-key>/cache.zip`. |
| [Docker](https://docs.gitlab.com/runner/executors/docker.html) | Locally, under [Docker volumes](https://docs.gitlab.com/runner/executors/docker.html#configure-directories-for-the-container-build-and-cache): `/var/lib/docker/volumes/<volume-id>/_data/<user>/<project>/<cache-key>/cache.zip`. |
| [Docker Machine](https://docs.gitlab.com/runner/executors/docker_machine.html) (autoscale runners) | The same as the Docker executor. |
If you use cache and artifacts to store the same path in your jobs, the cache might

View File

@ -48,8 +48,6 @@ To generate metric definitions, you can use the generator like this:
bin/rails g gitlab:analytics:internal_events \
--time_frames=7d 28d\
--group=project_management \
--stage=plan \
--section=dev \
--event=ci_templates_unique \
--unique=user.id \
--mr=https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121544

View File

@ -32,8 +32,6 @@ This example creates an event definition for an event called `project_created` a
bundle exec rails generate gitlab:analytics:internal_events \
--time_frames=7d 28d \
--group=project_management \
--stage=plan \
--section=dev \
--event=project_created \
--unique=user.id \
--mr=https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121544

View File

@ -20,6 +20,9 @@ See report and manage standards adherence, violations, and compliance frameworks
Standards adherence dashboard lists the adherence status of projects complying to GitLab standard.
When a project is added or an associated project or group setting is changed, an adherence scan is run on that project to update the standards adherence for that project. The date in the
**Last Scanned** column reflects any changes.
### View the standards adherence dashboard
Prerequisites:

View File

@ -352,6 +352,7 @@ To use SSH with GitLab, copy your public key to your GitLab account:
Verify that your SSH key was added correctly.
The following commands use the example hostname `gitlab.example.com`. Replace this example hostname with your GitLab instance's hostname, for example, `git@gitlab.com`.
By default, GitLab uses `git` username to authenticate. It can be different if it was [changed by the administrator](https://docs.gitlab.com/omnibus/settings/configuration.html#change-the-name-of-the-git-user-or-group).
1. To ensure you're connecting to the correct server, check the server's SSH host keys fingerprint. For:
- GitLab.com, see the [SSH host keys fingerprints](gitlab_com/index.md#ssh-host-keys-fingerprints) documentation.

View File

@ -7,8 +7,8 @@ property_description:
value_description:
extra_properties:
identifiers:<%= event_identifiers %>
product_section: <%= options.fetch(:section) %>
product_stage: <%= options.fetch(:stage) %>
product_section: <%= Gitlab::Analytics::GroupFetcher.section_text(options.fetch(:group)) %>
product_stage: <%= Gitlab::Analytics::GroupFetcher.stage_text(options.fetch(:group)) %>
product_group: <%= options.fetch(:group) %>
milestone: "<%= milestone %>"
introduced_by_url: <%= options.fetch(:mr) %>

View File

@ -1,8 +1,8 @@
---
key_path: <%= args.second %>
description: <%= args.last %>
product_section: <%= options.fetch(:section) %>
product_stage: <%= options.fetch(:stage) %>
product_section: <%= Gitlab::Analytics::GroupFetcher.section_text(options.fetch(:group)) %>
product_stage: <%= Gitlab::Analytics::GroupFetcher.stage_text(options.fetch(:group)) %>
product_group: <%= options.fetch(:group) %>
performance_indicator_type: []
value_type: number

View File

@ -0,0 +1,48 @@
# frozen_string_literal: true
module Gitlab
module Analytics
class GroupFetcher
class << self
def group_unknown?(group)
return false if groups.empty?
!groups.has_key?(group)
end
def stage_text(group)
groups[group]&.fetch(:stage) || ''
end
def section_text(group)
groups.dig(group, :section) || ''
end
private
# Output looks like { "import_and_integrate" => { stage: "manage", section: "dev" } ... }
# Returns {} if stages.yml cannot be fetched and parsed
def groups
return @groups if @groups
response = Gitlab::HTTP.get('https://gitlab.com/gitlab-com/www-gitlab-com/raw/master/data/stages.yml')
raise "Unable to load stages.yml" unless response.success?
data = YAML.safe_load(response.body)
groups_data = {}
data['stages'].each do |stage_name, stage_data|
stage_data['groups'].each_key do |group_name|
groups_data[group_name] = { stage: stage_name, section: stage_data['section'] }
end
end
@groups = groups_data.sort_by { |key, _value| key }.to_h
rescue StandardError
@groups = {}
end
end
end
end
end

View File

@ -50,15 +50,8 @@ module Gitlab
class_option :group,
type: :string,
optional: false,
desc: 'Name of group that added this metric'
class_option :stage,
type: :string,
optional: false,
desc: 'Name of stage that added this metric'
class_option :section,
type: :string,
optional: false,
desc: 'Name of section that added this metric'
desc: "Name of group that added this metric. " \
"See https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml"
class_option :mr,
type: :string,
optional: false,
@ -184,10 +177,12 @@ module Gitlab
def validate!
validate_tiers!
%i[event mr section stage group].each do |option|
%i[event mr group].each do |option|
raise "The option: --#{option} is missing" unless options.key? option
end
raise "Unknown group" if GroupFetcher.group_unknown?(options[:group])
time_frames.each do |time_frame|
validate_time_frame!(time_frame)

View File

@ -0,0 +1,100 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Analytics::GroupFetcher, :silence_stdout, feature_category: :service_ping do
let(:stage_data) do
<<~YAML
stages:
analyze:
section: analytics
groups:
analytics_instrumentation:
secure:
section: security
groups:
static_analysis:
dynamic_analysis:
YAML
end
let(:response) { instance_double(HTTParty::Response, success?: true, body: stage_data) }
around do |example|
described_class.instance_variable_set(:@groups, nil)
example.run
described_class.instance_variable_set(:@groups, nil)
end
before do
allow(Gitlab::HTTP).to receive(:get).and_return(response)
end
context 'when online' do
describe '.group_unknown?' do
it 'returns false for known groups' do
expect(described_class.group_unknown?('analytics_instrumentation')).to be_falsy
end
it 'returns true for unknown groups' do
expect(described_class.group_unknown?('unknown')).to be_truthy
end
end
describe '.stage_text' do
it 'returns the stage name for known groups' do
expect(described_class.stage_text('analytics_instrumentation')).to eq('analyze')
end
it 'returns empty string for unknown group' do
expect(described_class.stage_text('unknown')).to eq('')
end
end
describe '.section_text' do
it 'returns the section name for known groups' do
expect(described_class.section_text('analytics_instrumentation')).to eq('analytics')
end
it 'returns empty string for unknown group' do
expect(described_class.section_text('unknown')).to eq('')
end
end
end
context 'when offline' do
before do
allow(Gitlab::HTTP).to receive(:get).and_raise(Gitlab::HTTP_V2::BlockedUrlError)
end
describe '.group_unknown?' do
it 'returns false for known groups' do
expect(described_class.group_unknown?('analytics_instrumentation')).to be_falsy
end
it 'returns false for unknown group' do
expect(described_class.group_unknown?('unknown')).to be_falsy
end
end
describe '.stage_text' do
it 'returns empty string for known groups' do
expect(described_class.stage_text('analytics_instrumentation')).to eq('')
end
it 'returns empty string for unknown groups' do
expect(described_class.stage_text('unknown')).to eq('')
end
end
describe '.section_text' do
it 'returns empty string for known groups' do
expect(described_class.section_text('analytics_instrumentation')).to eq('')
end
it 'returns empty string for unknown groups' do
expect(described_class.section_text('unknown')).to eq('')
end
end
end
end

View File

@ -10,13 +10,14 @@ RSpec.describe Gitlab::Analytics::InternalEventsGenerator, :silence_stdout, feat
let(:tmpfile) { Tempfile.new('test-metadata') }
let(:existing_key_paths) { {} }
let(:description) { "This metric counts unique users viewing analytics metrics dashboard section" }
let(:group) { "group::analytics instrumentation" }
let(:stage) { "analytics" }
let(:group) { "analytics_instrumentation" }
let(:stage) { "analyze" }
let(:section) { "analytics" }
let(:mr) { "https://gitlab.com/some-group/some-project/-/merge_requests/123" }
let(:event) { "view_analytics_dashboard" }
let(:unique) { "user.id" }
let(:time_frames) { %w[7d] }
let(:group_unknown) { false }
let(:include_default_identifiers) { 'yes' }
let(:base_options) do
{
@ -24,8 +25,6 @@ RSpec.describe Gitlab::Analytics::InternalEventsGenerator, :silence_stdout, feat
free: true,
mr: mr,
group: group,
stage: stage,
section: section,
event: event,
unique: unique
}.stringify_keys
@ -45,6 +44,10 @@ RSpec.describe Gitlab::Analytics::InternalEventsGenerator, :silence_stdout, feat
.and_return(description)
end
allow(Gitlab::Analytics::GroupFetcher).to receive(:group_unknown?).and_return(group_unknown)
allow(Gitlab::Analytics::GroupFetcher).to receive(:stage_text).with(group).and_return(stage)
allow(Gitlab::Analytics::GroupFetcher).to receive(:section_text).with(group).and_return(section)
allow(Gitlab::TaskHelpers).to receive(:prompt).and_return(include_default_identifiers)
allow(Gitlab::Usage::MetricDefinition).to receive(:definitions).and_return(existing_key_paths)
end
@ -260,7 +263,7 @@ RSpec.describe Gitlab::Analytics::InternalEventsGenerator, :silence_stdout, feat
context 'without obligatory parameter' do
it 'raises error', :aggregate_failures do
%w[event mr section stage group].each do |option|
%w[event mr group].each do |option|
expect { described_class.new([], options.without(option)).invoke_all }
.to raise_error(RuntimeError)
end

View File

@ -14,13 +14,15 @@ RSpec.describe PipelineScheduleWorker, :sidekiq_inline, feature_category: :conti
create(:ci_pipeline_schedule, :nightly, project: project, owner: user)
end
let(:next_run_at) { pipeline_schedule.next_run_at }
before do
stub_application_setting(auto_devops_enabled: false)
stub_ci_pipeline_to_return_yaml_file
end
around do |example|
travel_to(pipeline_schedule.next_run_at + 1.hour) do
travel_to(next_run_at + 1.hour) do
example.run
end
end
@ -142,4 +144,52 @@ RSpec.describe PipelineScheduleWorker, :sidekiq_inline, feature_category: :conti
expect { subject }.to raise_error(Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError)
end
end
context 'with scheduling delay' do
before do
stub_const("#{described_class}::BATCH_SIZE", 1)
end
let!(:other_pipeline_schedule) do
create(:ci_pipeline_schedule, :every_minute, project: project, owner: user)
end
let(:next_run_at) do
[pipeline_schedule, other_pipeline_schedule].maximum(:next_run_at)
end
it 'calls bulk_perform_in with the arguments and delay' do
expect(RunPipelineScheduleWorker)
.to receive(:bulk_perform_in)
.with(7.seconds, [[pipeline_schedule.id, user.id, { scheduling: true }]])
.and_call_original
expect(RunPipelineScheduleWorker)
.to receive(:bulk_perform_in)
.with(14.seconds, [[other_pipeline_schedule.id, user.id, { scheduling: true }]])
.and_call_original
subject
end
context 'with run_pipeline_schedule_worker_with_delay disabled' do
before do
stub_feature_flags(run_pipeline_schedule_worker_with_delay: false)
end
it 'calls bulk_perform_async with the arguments and delay' do
expect(RunPipelineScheduleWorker)
.to receive(:bulk_perform_async)
.with([[pipeline_schedule.id, user.id, { scheduling: true }]])
.and_call_original
expect(RunPipelineScheduleWorker)
.to receive(:bulk_perform_async)
.with([[other_pipeline_schedule.id, user.id, { scheduling: true }]])
.and_call_original
subject
end
end
end
end