Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2021-03-30 09:10:51 +00:00
parent 0eeb173641
commit afa3d4e066
30 changed files with 738 additions and 359 deletions

View File

@ -3,13 +3,15 @@
module Packages
module Maven
class PackageFinder
attr_reader :path, :current_user, :project, :group
include ::Packages::FinderHelper
include Gitlab::Utils::StrongMemoize
def initialize(path, current_user, project: nil, group: nil)
def initialize(path, current_user, project: nil, group: nil, order_by_package_file: false)
@path = path
@current_user = current_user
@project = project
@group = group
@order_by_package_file = order_by_package_file
end
def execute
@ -23,9 +25,9 @@ module Packages
private
def base
if project
if @project
packages_for_a_single_project
elsif group
elsif @group
packages_for_multiple_projects
else
::Packages::Package.none
@ -33,8 +35,13 @@ module Packages
end
def packages_with_path
matching_packages = base.only_maven_packages_with_path(path, use_cte: group.present?)
matching_packages = matching_packages.order_by_package_file if versionless_package?(matching_packages)
matching_packages = base.only_maven_packages_with_path(@path, use_cte: @group.present?)
if group_level_improvements?
matching_packages = matching_packages.order_by_package_file if @order_by_package_file
else
matching_packages = matching_packages.order_by_package_file if versionless_package?(matching_packages)
end
matching_packages
end
@ -48,19 +55,29 @@ module Packages
# Produces a query that retrieves packages from a single project.
def packages_for_a_single_project
project.packages
@project.packages
end
# Produces a query that retrieves packages from multiple projects that
# the current user can view within a group.
def packages_for_multiple_projects
::Packages::Package.for_projects(projects_visible_to_current_user)
if group_level_improvements?
packages_visible_to_user(@current_user, within_group: @group)
else
::Packages::Package.for_projects(projects_visible_to_current_user)
end
end
# Returns the projects that the current user can view within a group.
def projects_visible_to_current_user
group.all_projects
.public_or_visible_to_user(current_user)
@group.all_projects
.public_or_visible_to_user(@current_user)
end
def group_level_improvements?
strong_memoize(:group_level_improvements) do
Feature.enabled?(:maven_packages_group_level_improvements)
end
end
end
end

View File

@ -136,7 +136,9 @@ class Packages::Package < ApplicationRecord
after_commit :update_composer_cache, on: :destroy, if: -> { composer? }
def self.for_projects(projects)
return none unless projects.any?
unless Feature.enabled?(:maven_packages_group_level_improvements)
return none unless projects.any?
end
where(project_id: projects)
end

View File

@ -84,13 +84,7 @@ class RemoteMirror < ApplicationRecord
end
after_transition started: :failed do |remote_mirror|
Gitlab::Metrics.add_event(:remote_mirrors_failed)
remote_mirror.update(last_update_at: Time.current)
remote_mirror.run_after_commit do
RemoteMirrorNotificationWorker.perform_async(remote_mirror.id)
end
remote_mirror.send_failure_notifications
end
end
@ -188,6 +182,24 @@ class RemoteMirror < ApplicationRecord
update_fail!
end
# Force the mrror into the retry state
def hard_retry!(error_message)
update_error_message(error_message)
self.update_status = :to_retry
save!(validate: false)
end
# Force the mirror into the failed state
def hard_fail!(error_message)
update_error_message(error_message)
self.update_status = :failed
save!(validate: false)
send_failure_notifications
end
def url=(value)
super(value) && return unless Gitlab::UrlSanitizer.valid?(value)
@ -239,6 +251,17 @@ class RemoteMirror < ApplicationRecord
last_update_at.present? ? MAX_INCREMENTAL_RUNTIME : MAX_FIRST_RUNTIME
end
def send_failure_notifications
Gitlab::Metrics.add_event(:remote_mirrors_failed)
run_after_commit do
RemoteMirrorNotificationWorker.perform_async(id)
end
self.last_update_at = Time.current
save!(validate: false)
end
private
def store_credentials

View File

@ -1851,10 +1851,12 @@ class User < ApplicationRecord
end
def dismissed_callout?(feature_name:, ignore_dismissal_earlier_than: nil)
callouts = self.callouts.with_feature_name(feature_name)
callouts = callouts.with_dismissed_after(ignore_dismissal_earlier_than) if ignore_dismissal_earlier_than
callout = callouts_by_feature_name[feature_name]
callouts.any?
return false unless callout
return callout.dismissed_after?(ignore_dismissal_earlier_than) if ignore_dismissal_earlier_than
true
end
# Load the current highest access by looking directly at the user's memberships
@ -1917,6 +1919,10 @@ class User < ApplicationRecord
private
def callouts_by_feature_name
@callouts_by_feature_name ||= callouts.index_by(&:feature_name)
end
def authorized_groups_without_shared_membership
Group.from_union([
groups,

View File

@ -38,6 +38,7 @@ class UserCallout < ApplicationRecord
uniqueness: { scope: :user_id },
inclusion: { in: UserCallout.feature_names.keys }
scope :with_feature_name, -> (feature_name) { where(feature_name: UserCallout.feature_names[feature_name]) }
scope :with_dismissed_after, -> (dismissed_after) { where('dismissed_at > ?', dismissed_after) }
def dismissed_after?(dismissed_after)
dismissed_at > dismissed_after
end
end

View File

@ -42,9 +42,9 @@ module Packages
def files
strong_memoize(:files) do
raise ExtractionError.new("is not a changes file") unless file_type == :changes
raise ExtractionError.new("Files field is missing") if fields[:Files].blank?
raise ExtractionError.new("Checksums-Sha1 field is missing") if fields[:'Checksums-Sha1'].blank?
raise ExtractionError.new("Checksums-Sha256 field is missing") if fields[:'Checksums-Sha256'].blank?
raise ExtractionError.new("Files field is missing") if fields['Files'].blank?
raise ExtractionError.new("Checksums-Sha1 field is missing") if fields['Checksums-Sha1'].blank?
raise ExtractionError.new("Checksums-Sha256 field is missing") if fields['Checksums-Sha256'].blank?
init_entries_from_files
entries_from_checksums_sha1
@ -56,7 +56,7 @@ module Packages
end
def init_entries_from_files
each_lines_for(:Files) do |line|
each_lines_for('Files') do |line|
md5sum, size, section, priority, filename = line.split
entry = FileEntry.new(
filename: filename,
@ -70,7 +70,7 @@ module Packages
end
def entries_from_checksums_sha1
each_lines_for(:'Checksums-Sha1') do |line|
each_lines_for('Checksums-Sha1') do |line|
sha1sum, size, filename = line.split
entry = @entries[filename]
raise ExtractionError.new("#{filename} is listed in Checksums-Sha1 but not in Files") unless entry
@ -81,7 +81,7 @@ module Packages
end
def entries_from_checksums_sha256
each_lines_for(:'Checksums-Sha256') do |line|
each_lines_for('Checksums-Sha256') do |line|
sha256sum, size, filename = line.split
entry = @entries[filename]
raise ExtractionError.new("#{filename} is listed in Checksums-Sha256 but not in Files") unless entry

View File

@ -72,7 +72,7 @@ module Packages
def extract_metadata
fields = extracted_fields
architecture = fields.delete(:Architecture) if file_type_debian?
architecture = fields.delete('Architecture') if file_type_debian?
{
file_type: file_type,

View File

@ -26,7 +26,7 @@ module Packages
section[field] += line[1..] unless paragraph_separator?(line)
elsif match = match_section_line(line)
section_name = match[:name] if section_name.nil?
field = match[:field].to_sym
field = match[:field]
raise InvalidDebian822Error, "Duplicate field '#{field}' in section '#{section_name}'" if section.include?(field)

View File

@ -9,8 +9,10 @@ module Projects
def execute(remote_mirror, tries)
return success unless remote_mirror.enabled?
# Blocked URLs are a hard failure, no need to attempt to retry
if Gitlab::UrlBlocker.blocked_url?(normalized_url(remote_mirror.url))
return error("The remote mirror URL is invalid.")
hard_retry_or_fail(remote_mirror, _('The remote mirror URL is invalid.'), tries)
return error(remote_mirror.last_error)
end
update_mirror(remote_mirror)
@ -19,11 +21,11 @@ module Projects
rescue Gitlab::Git::CommandError => e
# This happens if one of the gitaly calls above fail, for example when
# branches have diverged, or the pre-receive hook fails.
retry_or_fail(remote_mirror, e.message, tries)
hard_retry_or_fail(remote_mirror, e.message, tries)
error(e.message)
rescue => e
remote_mirror.mark_as_failed!(e.message)
remote_mirror.hard_fail!(e.message)
raise e
end
@ -70,15 +72,15 @@ module Projects
).execute
end
def retry_or_fail(mirror, message, tries)
def hard_retry_or_fail(mirror, message, tries)
if tries < MAX_TRIES
mirror.mark_for_retry!(message)
mirror.hard_retry!(message)
else
# It's not likely we'll be able to recover from this ourselves, so we'll
# notify the users of the problem, and don't trigger any sidekiq retries
# Instead, we'll wait for the next change to try the push again, or until
# a user manually retries.
mirror.mark_as_failed!(message)
mirror.hard_fail!(message)
end
end
end

View File

@ -126,7 +126,7 @@
.card
.card-header
= html_escape(_("%{group_name} group members")) % { group_name: "<strong>#{html_escape(@group.name)}</strong>".html_safe }
%span.badge.badge-pill= @group.members.size
%span.badge.badge-pill= @group.users_count
.float-right
= link_to group_group_members_path(@group), class: 'btn btn-default gl-button btn-sm' do
= sprite_icon('pencil-square', css_class: 'gl-icon')

View File

@ -0,0 +1,5 @@
---
title: A blocked URL for a push mirror is a hard failure
merge_request: 57392
author:
type: fixed

View File

@ -0,0 +1,5 @@
---
title: Make VALIDATION_REQUEST_TIMEOUT configurable
merge_request: 57521
author:
type: changed

View File

@ -0,0 +1,5 @@
---
title: Preload all user callouts in a single request
merge_request: 57679
author:
type: performance

View File

@ -0,0 +1,8 @@
---
name: maven_packages_group_level_improvements
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57600
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/326099
milestone: '13.11'
type: development
group: group::package
default_enabled: false

View File

@ -27,7 +27,15 @@ Response Code Legend:
## Configuration
Set the `EXTERNAL_VALIDATION_SERVICE_URL` to the external service URL and enable `ci_external_validation_service` feature flag.
To configure external pipeline validation:
1. Set the `EXTERNAL_VALIDATION_SERVICE_URL` environment variable to the external
service URL.
1. Enable the `ci_external_validation_service` feature flag.
By default, requests to the external service time out after five seconds. To override
the default, set the `EXTERNAL_VALIDATION_SERVICE_TIMEOUT` environment variable to the
required number of seconds.
## Payload Schema

View File

@ -77,6 +77,22 @@ module API
request.head? &&
file.fog_credentials[:provider] == 'AWS'
end
def fetch_package(file_name:, project: nil, group: nil)
order_by_package_file = false
if Feature.enabled?(:maven_packages_group_level_improvements)
order_by_package_file = file_name.include?(::Packages::Maven::Metadata.filename) &&
!params[:path].include?(::Packages::Maven::FindOrCreatePackageService::SNAPSHOT_TERM)
end
::Packages::Maven::PackageFinder.new(
params[:path],
current_user,
project: project,
group: group,
order_by_package_file: order_by_package_file
).execute!
end
end
desc 'Download the maven package file at instance level' do
@ -97,8 +113,7 @@ module API
authorize_read_package!(project)
package = ::Packages::Maven::PackageFinder
.new(params[:path], current_user, project: project).execute!
package = fetch_package(file_name: file_name, project: project)
package_file = ::Packages::PackageFileFinder
.new(package, file_name).execute!
@ -133,8 +148,7 @@ module API
not_found!('Group') unless can?(current_user, :read_group, group)
package = ::Packages::Maven::PackageFinder
.new(params[:path], current_user, group: group).execute!
package = fetch_package(file_name: file_name, group: group)
authorize_read_package!(package.project)
@ -171,8 +185,7 @@ module API
file_name, format = extract_format(params[:file_name])
package = ::Packages::Maven::PackageFinder
.new(params[:path], current_user, project: user_project).execute!
package = fetch_package(file_name: file_name, project: user_project)
package_file = ::Packages::PackageFileFinder
.new(package, file_name).execute!

View File

@ -10,7 +10,7 @@ module Gitlab
InvalidResponseCode = Class.new(StandardError)
VALIDATION_REQUEST_TIMEOUT = 5
DEFAULT_VALIDATION_REQUEST_TIMEOUT = 5
ACCEPTED_STATUS = 200
DOT_COM_REJECTED_STATUS = 406
GENERAL_REJECTED_STATUS = (400..499).freeze
@ -70,11 +70,18 @@ module Gitlab
def validate_service_request
Gitlab::HTTP.post(
validation_service_url, timeout: VALIDATION_REQUEST_TIMEOUT,
validation_service_url, timeout: validation_service_timeout,
body: validation_service_payload.to_json
)
end
def validation_service_timeout
timeout = ENV['EXTERNAL_VALIDATION_SERVICE_TIMEOUT'].to_i
return timeout if timeout > 0
DEFAULT_VALIDATION_REQUEST_TIMEOUT
end
def validation_service_url
ENV['EXTERNAL_VALIDATION_SERVICE_URL']
end

View File

@ -9,7 +9,7 @@ module Gitlab
production: "58dc0f06-936c-43b3-93bb-71693f1b6570"
}.freeze
UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{4}\h{8}/.freeze
UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{12}/.freeze
NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze
PACK_PATTERN = "NnnnnN"

View File

@ -30450,6 +30450,9 @@ msgstr ""
msgid "The regular expression used to find test coverage output in the job log. For example, use %{regex} for Simplecov (Ruby). Leave blank to disable."
msgstr ""
msgid "The remote mirror URL is invalid."
msgstr ""
msgid "The remote mirror took to long to complete."
msgstr ""

View File

@ -11,7 +11,8 @@ RSpec.describe ::Packages::Maven::PackageFinder do
let(:param_path) { nil }
let(:param_project) { nil }
let(:param_group) { nil }
let(:finder) { described_class.new(param_path, user, project: param_project, group: param_group) }
let(:param_order_by_package_file) { false }
let(:finder) { described_class.new(param_path, user, project: param_project, group: param_group, order_by_package_file: param_order_by_package_file) }
before do
group.add_developer(user)
@ -46,7 +47,23 @@ RSpec.describe ::Packages::Maven::PackageFinder do
context 'within a group' do
let(:param_group) { group }
it_behaves_like 'handling valid and invalid paths'
context 'with maven_packages_group_level_improvements enabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: true)
expect(finder).to receive(:packages_visible_to_user).with(user, within_group: group).and_call_original
end
it_behaves_like 'handling valid and invalid paths'
end
context 'with maven_packages_group_level_improvements disabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: false)
expect(finder).not_to receive(:packages_visible_to_user)
end
it_behaves_like 'handling valid and invalid paths'
end
end
context 'across all projects' do
@ -76,7 +93,39 @@ RSpec.describe ::Packages::Maven::PackageFinder do
create(:package_file, :xml, package: package2)
end
it { is_expected.to eq(package2) }
context 'with maven_packages_group_level_improvements enabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: true)
expect(finder).not_to receive(:versionless_package?)
end
context 'without order by package file' do
it { is_expected.to eq(package3) }
end
context 'with order by package file' do
let(:param_order_by_package_file) { true }
it { is_expected.to eq(package2) }
end
end
context 'with maven_packages_group_level_improvements disabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: false)
expect(finder).to receive(:versionless_package?).and_call_original
end
context 'without order by package file' do
it { is_expected.to eq(package2) }
end
context 'with order by package file' do
let(:param_order_by_package_file) { true }
it { is_expected.to eq(package2) }
end
end
end
end
end

View File

@ -62,11 +62,42 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::External do
it 'respects the defined payload schema' do
expect(::Gitlab::HTTP).to receive(:post) do |_url, params|
expect(params[:body]).to match_schema('/external_validation')
expect(params[:timeout]).to eq(described_class::DEFAULT_VALIDATION_REQUEST_TIMEOUT)
end
perform!
end
context 'with EXTERNAL_VALIDATION_SERVICE_TIMEOUT defined' do
before do
stub_env('EXTERNAL_VALIDATION_SERVICE_TIMEOUT', validation_service_timeout)
end
context 'with valid value' do
let(:validation_service_timeout) { '1' }
it 'uses defined timeout' do
expect(::Gitlab::HTTP).to receive(:post) do |_url, params|
expect(params[:timeout]).to eq(1)
end
perform!
end
end
context 'with invalid value' do
let(:validation_service_timeout) { '??' }
it 'uses default timeout' do
expect(::Gitlab::HTTP).to receive(:post) do |_url, params|
expect(params[:timeout]).to eq(described_class::DEFAULT_VALIDATION_REQUEST_TIMEOUT)
end
perform!
end
end
end
shared_examples 'successful external authorization' do
it 'does not drop the pipeline' do
perform!

View File

@ -99,6 +99,34 @@ RSpec.describe Packages::Package, type: :model do
end
end
describe '.for_projects' do
let_it_be(:package1) { create(:maven_package) }
let_it_be(:package2) { create(:maven_package) }
let_it_be(:package3) { create(:maven_package) }
let(:projects) { ::Project.id_in([package1.project_id, package2.project_id]) }
subject { described_class.for_projects(projects.select(:id)) }
it 'returns package1 and package2' do
expect(projects).not_to receive(:any?)
expect(subject).to match_array([package1, package2])
end
context 'with maven_packages_group_level_improvements disabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: false)
end
it 'returns package1 and package2' do
expect(projects).to receive(:any?).and_call_original
expect(subject).to match_array([package1, package2])
end
end
end
describe 'validations' do
subject { build(:package) }

View File

@ -263,6 +263,30 @@ RSpec.describe RemoteMirror, :mailer do
end
end
describe '#hard_retry!' do
let(:remote_mirror) { create(:remote_mirror).tap {|mirror| mirror.update_column(:url, 'invalid') } }
it 'transitions an invalid mirror to the to_retry state' do
remote_mirror.hard_retry!('Invalid')
expect(remote_mirror.update_status).to eq('to_retry')
expect(remote_mirror.last_error).to eq('Invalid')
end
end
describe '#hard_fail!' do
let(:remote_mirror) { create(:remote_mirror).tap {|mirror| mirror.update_column(:url, 'invalid') } }
it 'transitions an invalid mirror to the failed state' do
remote_mirror.hard_fail!('Invalid')
expect(remote_mirror.update_status).to eq('failed')
expect(remote_mirror.last_error).to eq('Invalid')
expect(remote_mirror.last_update_at).not_to be_nil
expect(RemoteMirrorNotificationWorker.jobs).not_to be_empty
end
end
context 'when remote mirror gets destroyed' do
it 'removes remote' do
mirror = create_mirror(url: 'http://foo:bar@test.com')

View File

@ -18,36 +18,14 @@ RSpec.describe UserCallout do
it { is_expected.to validate_uniqueness_of(:feature_name).scoped_to(:user_id).ignoring_case_sensitivity }
end
describe 'scopes' do
describe '.with_feature_name' do
let(:second_feature_name) { described_class.feature_names.keys.second }
let(:last_feature_name) { described_class.feature_names.keys.last }
describe '#dismissed_after?' do
let(:some_feature_name) { described_class.feature_names.keys.second }
let(:callout_dismissed_month_ago) { create(:user_callout, feature_name: some_feature_name, dismissed_at: 1.month.ago )}
let(:callout_dismissed_day_ago) { create(:user_callout, feature_name: some_feature_name, dismissed_at: 1.day.ago )}
it 'returns callout for requested feature name only' do
callout1 = create(:user_callout, feature_name: second_feature_name )
create(:user_callout, feature_name: last_feature_name )
callouts = described_class.with_feature_name(second_feature_name)
expect(callouts).to match_array([callout1])
end
end
describe '.with_dismissed_after' do
let(:some_feature_name) { described_class.feature_names.keys.second }
let(:callout_dismissed_month_ago) { create(:user_callout, feature_name: some_feature_name, dismissed_at: 1.month.ago )}
it 'does not return callouts dismissed before specified date' do
callouts = described_class.with_dismissed_after(15.days.ago)
expect(callouts).to match_array([])
end
it 'returns callouts dismissed after specified date' do
callouts = described_class.with_dismissed_after(2.months.ago)
expect(callouts).to match_array([callout_dismissed_month_ago])
end
it 'returns whether a callout dismissed after specified date' do
expect(callout_dismissed_month_ago.dismissed_after?(15.days.ago)).to eq(false)
expect(callout_dismissed_day_ago.dismissed_after?(15.days.ago)).to eq(true)
end
end
end

View File

@ -147,118 +147,136 @@ RSpec.describe API::MavenPackages do
end
describe 'GET /api/v4/packages/maven/*path/:file_name' do
context 'a public project' do
subject { download_file(package_file.file_name) }
shared_examples 'handling all conditions' do
context 'a public project' do
subject { download_file(package_file.file_name) }
it_behaves_like 'tracking the file download event'
it_behaves_like 'tracking the file download event'
it 'returns the file' do
subject
it 'returns the file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
it 'returns sha1 of the file' do
download_file(package_file.file_name + '.sha1')
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('text/plain')
expect(response.body).to eq(package_file.file_sha1)
end
end
it 'returns sha1 of the file' do
download_file(package_file.file_name + '.sha1')
context 'internal project' do
before do
project.team.truncate
project.update!(visibility_level: Gitlab::VisibilityLevel::INTERNAL)
end
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('text/plain')
expect(response.body).to eq(package_file.file_sha1)
subject { download_file_with_token(package_file.file_name) }
it_behaves_like 'tracking the file download event'
it 'returns the file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
it 'denies download when no private token' do
download_file(package_file.file_name)
expect(response).to have_gitlab_http_status(:forbidden)
end
it_behaves_like 'downloads with a job token'
it_behaves_like 'downloads with a deploy token'
end
context 'private project' do
subject { download_file_with_token(package_file.file_name) }
before do
project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
end
it_behaves_like 'tracking the file download event'
it 'returns the file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
it 'denies download when not enough permissions' do
project.add_guest(user)
subject
expect(response).to have_gitlab_http_status(:forbidden)
end
it 'denies download when no private token' do
download_file(package_file.file_name)
expect(response).to have_gitlab_http_status(:forbidden)
end
it_behaves_like 'downloads with a job token'
it_behaves_like 'downloads with a deploy token'
it 'does not allow download by a unauthorized deploy token with same id as a user with access' do
unauthorized_deploy_token = create(:deploy_token, read_package_registry: true, write_package_registry: true)
another_user = create(:user)
project.add_developer(another_user)
# We force the id of the deploy token and the user to be the same
unauthorized_deploy_token.update!(id: another_user.id)
download_file(
package_file.file_name,
{},
Gitlab::Auth::AuthFinders::DEPLOY_TOKEN_HEADER => unauthorized_deploy_token.token
)
expect(response).to have_gitlab_http_status(:forbidden)
end
end
context 'project name is different from a package name' do
before do
maven_metadatum.update!(path: "wrong_name/#{package.version}")
end
it 'rejects request' do
download_file(package_file.file_name)
expect(response).to have_gitlab_http_status(:forbidden)
end
end
end
context 'internal project' do
context 'with maven_packages_group_level_improvements enabled' do
before do
project.team.truncate
project.update!(visibility_level: Gitlab::VisibilityLevel::INTERNAL)
stub_feature_flags(maven_packages_group_level_improvements: true)
end
subject { download_file_with_token(package_file.file_name) }
it_behaves_like 'tracking the file download event'
it 'returns the file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
it 'denies download when no private token' do
download_file(package_file.file_name)
expect(response).to have_gitlab_http_status(:forbidden)
end
it_behaves_like 'downloads with a job token'
it_behaves_like 'downloads with a deploy token'
it_behaves_like 'handling all conditions'
end
context 'private project' do
subject { download_file_with_token(package_file.file_name) }
context 'with maven_packages_group_level_improvements disabled' do
before do
project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
stub_feature_flags(maven_packages_group_level_improvements: false)
end
it_behaves_like 'tracking the file download event'
it 'returns the file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
it 'denies download when not enough permissions' do
project.add_guest(user)
subject
expect(response).to have_gitlab_http_status(:forbidden)
end
it 'denies download when no private token' do
download_file(package_file.file_name)
expect(response).to have_gitlab_http_status(:forbidden)
end
it_behaves_like 'downloads with a job token'
it_behaves_like 'downloads with a deploy token'
it 'does not allow download by a unauthorized deploy token with same id as a user with access' do
unauthorized_deploy_token = create(:deploy_token, read_package_registry: true, write_package_registry: true)
another_user = create(:user)
project.add_developer(another_user)
# We force the id of the deploy token and the user to be the same
unauthorized_deploy_token.update!(id: another_user.id)
download_file(
package_file.file_name,
{},
Gitlab::Auth::AuthFinders::DEPLOY_TOKEN_HEADER => unauthorized_deploy_token.token
)
expect(response).to have_gitlab_http_status(:forbidden)
end
end
context 'project name is different from a package name' do
before do
maven_metadatum.update!(path: "wrong_name/#{package.version}")
end
it 'rejects request' do
download_file(package_file.file_name)
expect(response).to have_gitlab_http_status(:forbidden)
end
it_behaves_like 'handling all conditions'
end
def download_file(file_name, params = {}, request_headers = headers)
@ -274,6 +292,22 @@ RSpec.describe API::MavenPackages do
let(:url) { "/packages/maven/#{package.maven_metadatum.path}/#{package_file.file_name}" }
it_behaves_like 'processing HEAD requests'
context 'with maven_packages_group_level_improvements enabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: true)
end
it_behaves_like 'processing HEAD requests'
end
context 'with maven_packages_group_level_improvements disabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: false)
end
it_behaves_like 'processing HEAD requests'
end
end
describe 'GET /api/v4/groups/:id/-/packages/maven/*path/:file_name' do
@ -282,91 +316,11 @@ RSpec.describe API::MavenPackages do
group.add_developer(user)
end
context 'a public project' do
subject { download_file(package_file.file_name) }
shared_examples 'handling all conditions' do
context 'a public project' do
subject { download_file(package_file.file_name) }
it_behaves_like 'tracking the file download event'
it 'returns the file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
it 'returns sha1 of the file' do
download_file(package_file.file_name + '.sha1')
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('text/plain')
expect(response.body).to eq(package_file.file_sha1)
end
end
context 'internal project' do
before do
group.group_member(user).destroy!
project.update!(visibility_level: Gitlab::VisibilityLevel::INTERNAL)
end
subject { download_file_with_token(package_file.file_name) }
it_behaves_like 'tracking the file download event'
it 'returns the file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
it 'denies download when no private token' do
download_file(package_file.file_name)
expect(response).to have_gitlab_http_status(:not_found)
end
it_behaves_like 'downloads with a job token'
it_behaves_like 'downloads with a deploy token'
end
context 'private project' do
before do
project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
end
subject { download_file_with_token(package_file.file_name) }
it_behaves_like 'tracking the file download event'
it 'returns the file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
it 'denies download when not enough permissions' do
group.add_guest(user)
subject
expect(response).to have_gitlab_http_status(:forbidden)
end
it 'denies download when no private token' do
download_file(package_file.file_name)
expect(response).to have_gitlab_http_status(:not_found)
end
it_behaves_like 'downloads with a job token'
it_behaves_like 'downloads with a deploy token'
context 'with group deploy token' do
subject { download_file_with_token(package_file.file_name, {}, group_deploy_token_headers) }
it_behaves_like 'tracking the file download event'
it 'returns the file' do
subject
@ -375,15 +329,167 @@ RSpec.describe API::MavenPackages do
expect(response.media_type).to eq('application/octet-stream')
end
it 'returns the file with only write_package_registry scope' do
deploy_token_for_group.update!(read_package_registry: false)
it 'returns sha1 of the file' do
download_file(package_file.file_name + '.sha1')
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('text/plain')
expect(response.body).to eq(package_file.file_sha1)
end
end
context 'internal project' do
before do
group.group_member(user).destroy!
project.update!(visibility_level: Gitlab::VisibilityLevel::INTERNAL)
end
subject { download_file_with_token(package_file.file_name) }
it_behaves_like 'tracking the file download event'
it 'returns the file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
it 'denies download when no private token' do
download_file(package_file.file_name)
expect(response).to have_gitlab_http_status(:not_found)
end
it_behaves_like 'downloads with a job token'
it_behaves_like 'downloads with a deploy token'
end
context 'private project' do
before do
project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
end
subject { download_file_with_token(package_file.file_name) }
it_behaves_like 'tracking the file download event'
it 'returns the file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
it 'denies download when not enough permissions' do
group.add_guest(user)
subject
status = Feature.enabled?(:maven_packages_group_level_improvements) ? :not_found : :forbidden
expect(response).to have_gitlab_http_status(status)
end
it 'denies download when no private token' do
download_file(package_file.file_name)
expect(response).to have_gitlab_http_status(:not_found)
end
it_behaves_like 'downloads with a job token'
it_behaves_like 'downloads with a deploy token'
context 'with group deploy token' do
subject { download_file_with_token(package_file.file_name, {}, group_deploy_token_headers) }
it 'returns the file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
it 'returns the file with only write_package_registry scope' do
deploy_token_for_group.update!(read_package_registry: false)
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
end
end
context 'maven metadata file' do
let_it_be(:sub_group1) { create(:group, parent: group) }
let_it_be(:sub_group2) { create(:group, parent: group) }
let_it_be(:project1) { create(:project, :private, group: sub_group1) }
let_it_be(:project2) { create(:project, :private, group: sub_group2) }
let_it_be(:project3) { create(:project, :private, group: sub_group1) }
let_it_be(:package_name) { 'foo' }
let_it_be(:package1) { create(:maven_package, project: project1, name: package_name, version: nil) }
let_it_be(:package_file1) { create(:package_file, :xml, package: package1, file_name: 'maven-metadata.xml') }
let_it_be(:package2) { create(:maven_package, project: project2, name: package_name, version: nil) }
let_it_be(:package_file2) { create(:package_file, :xml, package: package2, file_name: 'maven-metadata.xml') }
let_it_be(:package3) { create(:maven_package, project: project3, name: package_name, version: nil) }
let_it_be(:package_file3) { create(:package_file, :xml, package: package3, file_name: 'maven-metadata.xml') }
let(:maven_metadatum) { package3.maven_metadatum }
subject { download_file_with_token(package_file3.file_name) }
before do
sub_group1.add_developer(user)
sub_group2.add_developer(user)
# the package with the most recently published file should be returned
create(:package_file, :xml, package: package2)
end
context 'in multiple versionless packages' do
it 'downloads the file' do
expect(::Packages::PackageFileFinder)
.to receive(:new).with(package2, 'maven-metadata.xml').and_call_original
subject
end
end
context 'in multiple snapshot packages' do
before do
version = '1.0.0-SNAPSHOT'
[package1, package2, package3].each do |pkg|
pkg.update!(version: version)
pkg.maven_metadatum.update!(path: "#{pkg.name}/#{pkg.version}")
end
end
it 'downloads the file' do
expect(::Packages::PackageFileFinder)
.to receive(:new).with(package3, 'maven-metadata.xml').and_call_original
subject
end
end
end
end
context 'with maven_packages_group_level_improvements enabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: true)
end
it_behaves_like 'handling all conditions'
end
context 'with maven_packages_group_level_improvements disabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: false)
end
it_behaves_like 'handling all conditions'
end
def download_file(file_name, params = {}, request_headers = headers)
@ -398,64 +504,96 @@ RSpec.describe API::MavenPackages do
describe 'HEAD /api/v4/groups/:id/-/packages/maven/*path/:file_name' do
let(:url) { "/groups/#{group.id}/-/packages/maven/#{package.maven_metadatum.path}/#{package_file.file_name}" }
it_behaves_like 'processing HEAD requests'
context 'with maven_packages_group_level_improvements enabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: true)
end
it_behaves_like 'processing HEAD requests'
end
context 'with maven_packages_group_level_improvements disabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: false)
end
it_behaves_like 'processing HEAD requests'
end
end
describe 'GET /api/v4/projects/:id/packages/maven/*path/:file_name' do
context 'a public project' do
subject { download_file(package_file.file_name) }
shared_examples 'handling all conditions' do
context 'a public project' do
subject { download_file(package_file.file_name) }
it_behaves_like 'tracking the file download event'
it_behaves_like 'tracking the file download event'
it 'returns the file' do
subject
it 'returns the file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
it 'returns sha1 of the file' do
download_file(package_file.file_name + '.sha1')
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('text/plain')
expect(response.body).to eq(package_file.file_sha1)
end
end
it 'returns sha1 of the file' do
download_file(package_file.file_name + '.sha1')
context 'private project' do
before do
project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
end
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('text/plain')
expect(response.body).to eq(package_file.file_sha1)
subject { download_file_with_token(package_file.file_name) }
it_behaves_like 'tracking the file download event'
it 'returns the file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
end
it 'denies download when not enough permissions' do
project.add_guest(user)
subject
expect(response).to have_gitlab_http_status(:forbidden)
end
it 'denies download when no private token' do
download_file(package_file.file_name)
expect(response).to have_gitlab_http_status(:not_found)
end
it_behaves_like 'downloads with a job token'
it_behaves_like 'downloads with a deploy token'
end
end
context 'private project' do
context 'with maven_packages_group_level_improvements enabled' do
before do
project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
stub_feature_flags(maven_packages_group_level_improvements: true)
end
subject { download_file_with_token(package_file.file_name) }
it_behaves_like 'handling all conditions'
end
it_behaves_like 'tracking the file download event'
it 'returns the file' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(response.media_type).to eq('application/octet-stream')
context 'with maven_packages_group_level_improvements disabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: false)
end
it 'denies download when not enough permissions' do
project.add_guest(user)
subject
expect(response).to have_gitlab_http_status(:forbidden)
end
it 'denies download when no private token' do
download_file(package_file.file_name)
expect(response).to have_gitlab_http_status(:not_found)
end
it_behaves_like 'downloads with a job token'
it_behaves_like 'downloads with a deploy token'
it_behaves_like 'handling all conditions'
end
def download_file(file_name, params = {}, request_headers = headers)
@ -471,7 +609,21 @@ RSpec.describe API::MavenPackages do
describe 'HEAD /api/v4/projects/:id/packages/maven/*path/:file_name' do
let(:url) { "/projects/#{project.id}/packages/maven/#{package.maven_metadatum.path}/#{package_file.file_name}" }
it_behaves_like 'processing HEAD requests'
context 'with maven_packages_group_level_improvements enabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: true)
end
it_behaves_like 'processing HEAD requests'
end
context 'with maven_packages_group_level_improvements disabled' do
before do
stub_feature_flags(maven_packages_group_level_improvements: false)
end
it_behaves_like 'processing HEAD requests'
end
end
describe 'PUT /api/v4/projects/:id/packages/maven/*path/:file_name/authorize' do

View File

@ -13,7 +13,7 @@ RSpec.describe Packages::Debian::ExtractChangesMetadataService do
context 'with valid package file' do
it 'extract metadata', :aggregate_failures do
expected_fields = { 'Architecture': 'source amd64', 'Binary': 'libsample0 sample-dev sample-udeb' }
expected_fields = { 'Architecture' => 'source amd64', 'Binary' => 'libsample0 sample-dev sample-udeb' }
expect(subject[:file_type]).to eq(:changes)
expect(subject[:architecture]).to be_nil
@ -40,7 +40,7 @@ RSpec.describe Packages::Debian::ExtractChangesMetadataService do
let(:sha256_dsc) { '844f79825b7e8aaa191e514b58a81f9ac1e58e2180134b0c9512fa66d896d7ba 671 sample_1.2.3~alpha2.dsc' }
let(:sha256_source) { 'b5a599e88e7cbdda3bde808160a21ba1dd1ec76b2ec8d4912aae769648d68362 864 sample_1.2.3~alpha2.tar.xz' }
let(:sha256s) { "#{sha256_dsc}\n#{sha256_source}" }
let(:fields) { { Files: md5s, 'Checksums-Sha1': sha1s, 'Checksums-Sha256': sha256s } }
let(:fields) { { 'Files' => md5s, 'Checksums-Sha1' => sha1s, 'Checksums-Sha256' => sha256s } }
let(:metadata) { { file_type: :changes, architecture: 'amd64', fields: fields } }
before do

View File

@ -10,17 +10,17 @@ RSpec.describe Packages::Debian::ExtractDebMetadataService do
context 'with correct file' do
it 'return as expected' do
expected = {
'Package': 'libsample0',
'Source': 'sample',
'Version': '1.2.3~alpha2',
'Architecture': 'amd64',
'Maintainer': 'John Doe <john.doe@example.com>',
'Installed-Size': '7',
'Section': 'libs',
'Priority': 'optional',
'Multi-Arch': 'same',
'Homepage': 'https://gitlab.com/',
'Description': "Some mostly empty lib\nUsed in GitLab tests.\n\nTesting another paragraph."
'Package' => 'libsample0',
'Source' => 'sample',
'Version' => '1.2.3~alpha2',
'Architecture' => 'amd64',
'Maintainer' => 'John Doe <john.doe@example.com>',
'Installed-Size' => '7',
'Section' => 'libs',
'Priority' => 'optional',
'Multi-Arch' => 'same',
'Homepage' => 'https://gitlab.com/',
'Description' => "Some mostly empty lib\nUsed in GitLab tests.\n\nTesting another paragraph."
}
expect(subject.execute).to eq expected

View File

@ -33,11 +33,11 @@ RSpec.describe Packages::Debian::ExtractMetadataService do
where(:case_name, :trait, :expected_file_type, :expected_architecture, :expected_fields) do
'with invalid' | :invalid | :unknown | nil | nil
'with source' | :source | :source | nil | nil
'with dsc' | :dsc | :dsc | nil | { 'Binary': 'sample-dev, libsample0, sample-udeb' }
'with deb' | :deb | :deb | 'amd64' | { 'Multi-Arch': 'same' }
'with udeb' | :udeb | :udeb | 'amd64' | { 'Package': 'sample-udeb' }
'with buildinfo' | :buildinfo | :buildinfo | nil | { 'Architecture': 'amd64 source', 'Build-Architecture': 'amd64' }
'with changes' | :changes | :changes | nil | { 'Architecture': 'source amd64', 'Binary': 'libsample0 sample-dev sample-udeb' }
'with dsc' | :dsc | :dsc | nil | { 'Binary' => 'sample-dev, libsample0, sample-udeb' }
'with deb' | :deb | :deb | 'amd64' | { 'Multi-Arch' => 'same' }
'with udeb' | :udeb | :udeb | 'amd64' | { 'Package' => 'sample-udeb' }
'with buildinfo' | :buildinfo | :buildinfo | nil | { 'Architecture' => 'amd64 source', 'Build-Architecture' => 'amd64' }
'with changes' | :changes | :changes | nil | { 'Architecture' => 'source amd64', 'Binary' => 'libsample0 sample-dev sample-udeb' }
end
with_them do

View File

@ -27,17 +27,17 @@ RSpec.describe Packages::Debian::ParseDebian822Service do
it 'return as expected, preserving order' do
expected = {
'Package: libsample0' => {
'Package': 'libsample0',
'Source': 'sample',
'Version': '1.2.3~alpha2',
'Architecture': 'amd64',
'Maintainer': 'John Doe <john.doe@example.com>',
'Installed-Size': '9',
'Section': 'libs',
'Priority': 'optional',
'Multi-Arch': 'same',
'Homepage': 'https://gitlab.com/',
'Description': "Some mostly empty lib\nUsed in GitLab tests.\n\nTesting another paragraph."
'Package' => 'libsample0',
'Source' => 'sample',
'Version' => '1.2.3~alpha2',
'Architecture' => 'amd64',
'Maintainer' => 'John Doe <john.doe@example.com>',
'Installed-Size' => '9',
'Section' => 'libs',
'Priority' => 'optional',
'Multi-Arch' => 'same',
'Homepage' => 'https://gitlab.com/',
'Description' => "Some mostly empty lib\nUsed in GitLab tests.\n\nTesting another paragraph."
}
}
@ -51,38 +51,38 @@ RSpec.describe Packages::Debian::ParseDebian822Service do
it 'return as expected, preserving order' do
expected = {
'Source: sample' => {
'Source': 'sample',
'Priority': 'optional',
'Maintainer': 'John Doe <john.doe@example.com>',
'Build-Depends': 'debhelper-compat (= 13)',
'Standards-Version': '4.5.0',
'Section': 'libs',
'Homepage': 'https://gitlab.com/',
# 'Vcs-Browser': 'https://salsa.debian.org/debian/sample-1.2.3',
# '#Vcs-Git': 'https://salsa.debian.org/debian/sample-1.2.3.git',
'Rules-Requires-Root': 'no'
'Source' => 'sample',
'Priority' => 'optional',
'Maintainer' => 'John Doe <john.doe@example.com>',
'Build-Depends' => 'debhelper-compat (= 13)',
'Standards-Version' => '4.5.0',
'Section' => 'libs',
'Homepage' => 'https://gitlab.com/',
# 'Vcs-Browser' => 'https://salsa.debian.org/debian/sample-1.2.3',
# '#Vcs-Git' => 'https://salsa.debian.org/debian/sample-1.2.3.git',
'Rules-Requires-Root' => 'no'
},
'Package: sample-dev' => {
'Package': 'sample-dev',
'Section': 'libdevel',
'Architecture': 'any',
'Multi-Arch': 'same',
'Depends': 'libsample0 (= ${binary:Version}), ${misc:Depends}',
'Description': "Some mostly empty developpement files\nUsed in GitLab tests.\n\nTesting another paragraph."
'Package' => 'sample-dev',
'Section' => 'libdevel',
'Architecture' => 'any',
'Multi-Arch' => 'same',
'Depends' => 'libsample0 (= ${binary:Version}), ${misc:Depends}',
'Description' => "Some mostly empty developpement files\nUsed in GitLab tests.\n\nTesting another paragraph."
},
'Package: libsample0' => {
'Package': 'libsample0',
'Architecture': 'any',
'Multi-Arch': 'same',
'Depends': '${shlibs:Depends}, ${misc:Depends}',
'Description': "Some mostly empty lib\nUsed in GitLab tests.\n\nTesting another paragraph."
'Package' => 'libsample0',
'Architecture' => 'any',
'Multi-Arch' => 'same',
'Depends' => '${shlibs:Depends}, ${misc:Depends}',
'Description' => "Some mostly empty lib\nUsed in GitLab tests.\n\nTesting another paragraph."
},
'Package: sample-udeb' => {
'Package': 'sample-udeb',
'Package-Type': 'udeb',
'Architecture': 'any',
'Depends': 'installed-base',
'Description': 'Some mostly empty udeb'
'Package' => 'sample-udeb',
'Package-Type' => 'udeb',
'Architecture' => 'any',
'Depends' => 'installed-base',
'Description' => 'Some mostly empty udeb'
}
}

View File

@ -12,7 +12,9 @@ RSpec.describe Projects::UpdateRemoteMirrorService do
subject(:service) { described_class.new(project, project.creator) }
describe '#execute' do
subject(:execute!) { service.execute(remote_mirror, 0) }
let(:retries) { 0 }
subject(:execute!) { service.execute(remote_mirror, retries) }
before do
project.repository.add_branch(project.owner, 'existing-branch', 'master')
@ -62,8 +64,18 @@ RSpec.describe Projects::UpdateRemoteMirrorService do
allow(Gitlab::UrlBlocker).to receive(:blocked_url?).and_return(true)
end
it 'fails and returns error status' do
it 'hard retries and returns error status' do
expect(execute!).to eq(status: :error, message: 'The remote mirror URL is invalid.')
expect(remote_mirror).to be_to_retry
end
context 'when retries are exceeded' do
let(:retries) { 4 }
it 'hard fails and returns error status' do
expect(execute!).to eq(status: :error, message: 'The remote mirror URL is invalid.')
expect(remote_mirror).to be_failed
end
end
end