root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-11-10 09:11:08 +00:00
parent ec890a64f7
commit 1f64fe671b
33 changed files with 483 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/assets/javascripts/ci_variable_list/components/ci_variable_modal.vue
View File

@ -293,7 +293,7 @@ export default {
v-model="variable.value"
:state="variableValidationState"
rows="3"
max-rows="6"
max-rows="10"
data-testid="pipeline-form-ci-variable-value"
data-qa-selector="ci_variable_value_field"
class="gl-font-monospace!"

2
app/assets/javascripts/ci_variable_list/components/legacy_ci_variable_modal.vue
View File

@ -271,7 +271,7 @@ export default {
v-model="secret_value"
:state="variableValidationState"
rows="3"
max-rows="6"
max-rows="10"
data-testid="pipeline-form-ci-variable-value"
data-qa-selector="ci_variable_value_field"
class="gl-font-monospace!"

15
app/assets/javascripts/environments/components/environment_rollback.vue
View File

@ -51,17 +51,20 @@ export default {
methods: {
onClick() {
const rollbackEnvironmentData = {
...this.environment,
retryUrl: this.retryUrl,
isLastDeployment: this.isLastDeployment,
};
if (this.graphql) {
this.$apollo.mutate({
mutation: setEnvironmentToRollback,
variables: { environment: this.environment },
variables: {
environment: rollbackEnvironmentData,
},
});
} else {
eventHub.$emit('requestRollbackEnvironment', {
...this.environment,
retryUrl: this.retryUrl,
isLastDeployment: this.isLastDeployment,
});
eventHub.$emit('requestRollbackEnvironment', rollbackEnvironmentData);
}
},
},

1
app/assets/javascripts/environments/graphql/queries/environment_to_rollback.query.graphql
View File

@ -3,5 +3,6 @@ query environmentToRollback {
id
name
lastDeployment
retryUrl
}
}

12
app/controllers/concerns/send_file_upload.rb
View File

@ -30,7 +30,17 @@ module SendFileUpload
headers.store(*Gitlab::Workhorse.send_url(file_upload.url(**redirect_params)))
head :ok
else
redirect_to file_upload.url(**redirect_params)
redirect_to cdn_fronted_url(file_upload, redirect_params)
end
end
def cdn_fronted_url(file, redirect_params)
if Feature.enabled?(:use_cdn_with_job_artifacts_ui_downloads) && file.respond_to?(:cdn_enabled_url)
result = file.cdn_enabled_url(request.remote_ip, redirect_params[:query])
Gitlab::ApplicationContext.push(artifact_used_cdn: result.used_cdn)
result.url
else
file.url(**redirect_params)
end
end

8
app/uploaders/object_storage/cdn.rb
View File

@ -12,9 +12,9 @@ module ObjectStorage
UrlResult = Struct.new(:url, :used_cdn)
def cdn_enabled_url(ip_address)
def cdn_enabled_url(ip_address, params = {})
if use_cdn?(ip_address)
UrlResult.new(cdn_signed_url, true)
UrlResult.new(cdn_signed_url(params), true)
else
UrlResult.new(url, false)
end
@ -27,8 +27,8 @@ module ObjectStorage
cdn_provider.use_cdn?(request_ip)
end
def cdn_signed_url
cdn_provider&.signed_url(path)
def cdn_signed_url(params = {})
cdn_provider&.signed_url(path, params: params)
end
private

16
app/uploaders/object_storage/cdn/google_cdn.rb
View File

@ -24,18 +24,24 @@ module ObjectStorage
!GoogleIpCache.google_ip?(request_ip)
end
def signed_url(path, expiry: 10.minutes)
def signed_url(path, expiry: 10.minutes, params: {})
expiration = (Time.current + expiry).utc.to_i
uri = Addressable::URI.parse(cdn_url)
uri.path = path
uri.query = "Expires=#{expiration}&KeyName=#{key_name}"
# Use an Array to preserve order: Google CDN needs to have
# Expires, KeyName, and Signature in that order or it will return a 403 error:
# https://cloud.google.com/cdn/docs/troubleshooting-steps#signing
query_params = params.to_a
query_params << ['Expires', expiration]
query_params << ['KeyName', key_name]
uri.query_values = query_params
signature = OpenSSL::HMAC.digest('SHA1', decoded_key, uri.to_s)
unsigned_url = uri.to_s
signature = OpenSSL::HMAC.digest('SHA1', decoded_key, unsigned_url)
encoded_signature = Base64.urlsafe_encode64(signature)
uri.query += "&Signature=#{encoded_signature}"
uri.to_s
"#{unsigned_url}&Signature=#{encoded_signature}"
end
private

8
config/feature_flags/development/use_cdn_with_job_artifacts_ui_downloads.yml Normal file
View File

@ -0,0 +1,8 @@
---
name: use_cdn_with_job_artifacts_ui_downloads
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/102839
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/381479
milestone: '15.6'
type: development
group: group::pipeline insights
default_enabled: false

22
db/structure.sql
View File

@ -225,15 +225,6 @@ RETURN NULL;
END
$$;
CREATE FUNCTION trigger_1a857e8db6cd() RETURNS trigger
LANGUAGE plpgsql
AS $$
BEGIN
NEW."uuid_convert_string_to_uuid" := NEW."uuid";
RETURN NEW;
END;
$$;
CREATE FUNCTION sync_namespaces_amount_used_columns() RETURNS trigger
LANGUAGE plpgsql
AS $$
@ -252,6 +243,15 @@ BEGIN
END;
$$;
CREATE FUNCTION trigger_1a857e8db6cd() RETURNS trigger
LANGUAGE plpgsql
AS $$
BEGIN
NEW."uuid_convert_string_to_uuid" := NEW."uuid";
RETURN NEW;
END;
$$;
CREATE FUNCTION unset_has_issues_on_vulnerability_reads() RETURNS trigger
LANGUAGE plpgsql
AS $$
@ -32566,12 +32566,12 @@ CREATE TRIGGER nullify_merge_request_metrics_build_data_on_update BEFORE UPDATE
CREATE TRIGGER projects_loose_fk_trigger AFTER DELETE ON projects REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records();
CREATE TRIGGER trigger_1a857e8db6cd BEFORE INSERT OR UPDATE ON vulnerability_occurrences FOR EACH ROW EXECUTE FUNCTION trigger_1a857e8db6cd();
CREATE TRIGGER sync_namespaces_amount_used_columns BEFORE INSERT OR UPDATE ON ci_namespace_monthly_usages FOR EACH ROW EXECUTE FUNCTION sync_namespaces_amount_used_columns();
CREATE TRIGGER sync_projects_amount_used_columns BEFORE INSERT OR UPDATE ON ci_project_monthly_usages FOR EACH ROW EXECUTE FUNCTION sync_projects_amount_used_columns();
CREATE TRIGGER trigger_1a857e8db6cd BEFORE INSERT OR UPDATE ON vulnerability_occurrences FOR EACH ROW EXECUTE FUNCTION trigger_1a857e8db6cd();
CREATE TRIGGER trigger_delete_project_namespace_on_project_delete AFTER DELETE ON projects FOR EACH ROW WHEN ((old.project_namespace_id IS NOT NULL)) EXECUTE FUNCTION delete_associated_project_namespace();
CREATE TRIGGER trigger_has_external_issue_tracker_on_delete AFTER DELETE ON integrations FOR EACH ROW WHEN ((((old.category)::text = 'issue_tracker'::text) AND (old.active = true) AND (old.project_id IS NOT NULL))) EXECUTE FUNCTION set_has_external_issue_tracker();

7
doc/development/documentation/topic_types/tutorial.md
View File

@ -58,6 +58,13 @@ To do step 2:
Start the page title with `Tutorial:` followed by an active verb, like `Tutorial: Create a website`.
In the left nav, use the full page title. Do not abbreviate it.
Put the text in quotes so the pipeline will pass. For example,
`"Tutorial: Make your first Git commit"`.
On [the **Learn GitLab with tutorials** page](../../../tutorials/index.md),
do not use `Tutorial` in the title.
## Screenshots
You can include screenshots in a tutorial to illustrate important steps in the process.

6
doc/development/fe_guide/graphql.md
View File

@ -907,7 +907,7 @@ For example, we have a query like this:
query searchGroupsWhereUserCanTransfer {
currentUser {
id
groups {
groups(after: 'somecursor') {
nodes {
id
fullName
@ -920,9 +920,7 @@ query searchGroupsWhereUserCanTransfer {
}
```
Here, the `groups` field doesn't have a good candidate for `keyArgs`: both
`nodes` and `pageInfo` will be updated when we're fetching a second page.
Setting `keyArgs` to `false` makes the update work as intended:
Here, the `groups` field doesn't have a good candidate for `keyArgs`: we don't want to account for `after` argument because it will change on requesting subsequent pages. Setting `keyArgs` to `false` makes the update work as intended:
```javascript
typePolicies: {

10
doc/user/project/remote_development/index.md
View File

@ -6,6 +6,14 @@ info: To determine the technical writer assigned to the Stage/Group associated w
# Remote Development **(FREE)**
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95169) in GitLab 15.6 [with a flag](../../../administration/feature_flags.md) named `vscode_web_ide`. Disabled by default.
FLAG:
On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `vscode_web_ide`. On GitLab.com, this feature is available. The feature is not ready for production use.
WARNING:
This feature is in [Alpha](../../../policy/alpha-beta-support.md#alpha-features) and subject to change without notice.
DISCLAIMER:
This page contains information related to upcoming products, features, and functionality.
It is important to note that the information presented is for informational purposes only.
@ -89,7 +97,7 @@ docker run -d \
-v "${CERTS_DIR}/fullchain.pem:/gitlab-rd-web-ide/certs/fullchain.pem" \
-v "${CERTS_DIR}/privkey.pem:/gitlab-rd-web-ide/certs/privkey.pem" \
-v "${PROJECTS_DIR}:/projects" \
registry.gitlab.com/gitlab-com/create-stage/editor-poc/remote-development/gitlab-rd-web-ide-docker:0.1 \
registry.gitlab.com/gitlab-com/create-stage/editor-poc/remote-development/gitlab-rd-web-ide-docker:0.1-alpha \
--log-level warn --domain "${DOMAIN}" --ignore-version-mismatch
```

58
lib/gitlab/database/migration_helpers.rb
View File

@ -9,6 +9,7 @@ module Gitlab
include Migrations::LockRetriesHelpers
include Migrations::TimeoutHelpers
include Migrations::ConstraintsHelpers
include Migrations::ExtensionHelpers
include DynamicModelHelpers
include RenameTableHelpers
include AsyncIndexes::MigrationHelpers
@ -1136,63 +1137,6 @@ into similar problems in the future (e.g. when new tables are created).
execute(sql)
end
def create_extension(extension)
execute('CREATE EXTENSION IF NOT EXISTS %s' % extension)
rescue ActiveRecord::StatementInvalid => e
dbname = ApplicationRecord.database.database_name
user = ApplicationRecord.database.username
warn(<<~MSG) if e.to_s =~ /permission denied/
GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but
the database user is not allowed to install the extension.
You can either install the extension manually using a database superuser:
CREATE EXTENSION IF NOT EXISTS #{extension}
Or, you can solve this by logging in to the GitLab
database (#{dbname}) using a superuser and running:
ALTER #{user} WITH SUPERUSER
This query will grant the user superuser permissions, ensuring any database extensions
can be installed through migrations.
For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html.
MSG
raise
end
def drop_extension(extension)
execute('DROP EXTENSION IF EXISTS %s' % extension)
rescue ActiveRecord::StatementInvalid => e
dbname = ApplicationRecord.database.database_name
user = ApplicationRecord.database.username
warn(<<~MSG) if e.to_s =~ /permission denied/
This migration attempts to drop the PostgreSQL extension '#{extension}'
installed in database '#{dbname}', but the database user is not allowed
to drop the extension.
You can either drop the extension manually using a database superuser:
DROP EXTENSION IF EXISTS #{extension}
Or, you can solve this by logging in to the GitLab
database (#{dbname}) using a superuser and running:
ALTER #{user} WITH SUPERUSER
This query will grant the user superuser permissions, ensuring any database extensions
can be dropped through migrations.
For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html.
MSG
raise
end
def add_primary_key_using_index(table_name, pk_name, index_to_use)
execute <<~SQL
ALTER TABLE #{quote_table_name(table_name)} ADD CONSTRAINT #{quote_table_name(pk_name)} PRIMARY KEY USING INDEX #{quote_table_name(index_to_use)}

66
lib/gitlab/database/migrations/extension_helpers.rb Normal file
View File

@ -0,0 +1,66 @@
# frozen_string_literal: true
module Gitlab
module Database
module Migrations
module ExtensionHelpers
def create_extension(extension)
execute("CREATE EXTENSION IF NOT EXISTS #{extension}")
rescue ActiveRecord::StatementInvalid => e
dbname = ApplicationRecord.database.database_name
user = ApplicationRecord.database.username
warn(<<~MSG) if e.to_s.include?('permission denied')
GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but
the database user is not allowed to install the extension.
You can either install the extension manually using a database superuser:
CREATE EXTENSION IF NOT EXISTS #{extension}
Or, you can solve this by logging in to the GitLab
database (#{dbname}) using a superuser and running:
ALTER #{user} WITH SUPERUSER
This query will grant the user superuser permissions, ensuring any database extensions
can be installed through migrations.
For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html.
MSG
raise
end
def drop_extension(extension)
execute("DROP EXTENSION IF EXISTS #{extension}")
rescue ActiveRecord::StatementInvalid => e
dbname = ApplicationRecord.database.database_name
user = ApplicationRecord.database.username
warn(<<~MSG) if e.to_s.include?('permission denied')
This migration attempts to drop the PostgreSQL extension '#{extension}'
installed in database '#{dbname}', but the database user is not allowed
to drop the extension.
You can either drop the extension manually using a database superuser:
DROP EXTENSION IF EXISTS #{extension}
Or, you can solve this by logging in to the GitLab
database (#{dbname}) using a superuser and running:
ALTER #{user} WITH SUPERUSER
This query will grant the user superuser permissions, ensuring any database extensions
can be dropped through migrations.
For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html.
MSG
raise
end
end
end
end
end

2
qa/Gemfile
View File

@ -14,7 +14,7 @@ gem 'airborne', '~> 0.3.7', require: false # airborne is messing with rspec sand
gem 'rest-client', '~> 2.1.0'
gem 'rspec-retry', '~> 0.6.2', require: 'rspec/retry'
gem 'rspec_junit_formatter', '~> 0.6.0'
gem 'faker', '~> 2.23'
gem 'faker', '~> 3.0'
gem 'knapsack', '~> 4.0'
gem 'parallel_tests', '~> 4.0'
gem 'rotp', '~> 6.2.0'

4
qa/Gemfile.lock
View File

@ -61,7 +61,7 @@ GEM
domain_name (0.5.20190701)
unf (>= 0.0.5, < 1.0.0)
excon (0.92.4)
faker (2.23.0)
faker (3.0.0)
i18n (>= 1.8.11, < 2)
faraday (2.5.2)
faraday-net_http (>= 2.0, < 3.1)
@ -307,7 +307,7 @@ DEPENDENCIES
chemlab-library-www-gitlab-com (~> 0.1, >= 0.1.1)
confiner (~> 0.3)
deprecation_toolkit (~> 2.0.0)
faker (~> 2.23)
faker (~> 3.0)
faraday-retry (~> 2.0)
fog-core (= 2.1.0)
fog-google (~> 1.19)

5
qa/qa/page/component/namespace_select.rb
View File

@ -25,7 +25,10 @@ module QA
wait_for_requests
click_element(:namespaces_list_item, text: item)
# Click element by JS in case dropdown changes position mid-click
# Workaround for issue https://gitlab.com/gitlab-org/gitlab/-/issues/381376
namespace = find_element(:namespaces_list_item, text: item, visible: false)
click_by_javascript(namespace)
end
end
end

10
qa/qa/page/project/settings/advanced.rb
View File

@ -53,11 +53,7 @@ module QA
def transfer_project!(project_name, namespace)
QA::Runtime::Logger.info "Transferring project: #{project_name} to namespace: #{namespace}"
wait_for_transfer_project_content
# Scroll to bottom of page to prevent namespace dropdown from changing position mid-click
# See https://gitlab.com/gitlab-org/gitlab/-/issues/381376 for details
page.scroll_to(:bottom)
scroll_to_transfer_project_content
# Workaround for a failure to search when there are no spaces around the /
# https://gitlab.com/gitlab-org/gitlab/-/issues/218965
@ -102,10 +98,12 @@ module QA
private
def wait_for_transfer_project_content
def scroll_to_transfer_project_content
retry_until(sleep_interval: 1, message: 'Waiting for transfer project content to display') do
has_element?(:transfer_project_content, wait: 3)
end
scroll_to_element :transfer_project_content
end
def wait_for_enabled_transfer_project_button

4
qa/qa/specs/features/browser_ui/4_verify/ci_variable/pipeline_with_protected_variable_spec.rb
View File

@ -3,8 +3,8 @@
module QA
RSpec.describe 'Verify', :runner, product_group: :pipeline_authoring do
describe 'Pipeline with protected variable' do
let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" }
let(:protected_value) { Faker::Alphanumeric.alphanumeric(8) }
let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" }
let(:protected_value) { Faker::Alphanumeric.alphanumeric(number: 8) }
let(:project) do
Resource::Project.fabricate_via_api! do |project|

2
qa/qa/specs/features/browser_ui/4_verify/pipeline/include_multiple_files_from_a_project_spec.rb
View File

@ -3,7 +3,7 @@
module QA
RSpec.describe 'Verify', :runner, product_group: :pipeline_authoring do
describe 'Include multiple files from a project' do
let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" }
let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" }
let(:expected_text) { Faker::Lorem.sentence }
let(:unexpected_text) { Faker::Lorem.sentence }

2
qa/qa/specs/features/browser_ui/4_verify/pipeline/merge_mr_when_pipline_is_blocked_spec.rb
View File

@ -3,7 +3,7 @@
module QA
RSpec.describe 'Verify', :runner, product_group: :pipeline_execution do
context 'When pipeline is blocked' do
let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" }
let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" }
let(:project) do
Resource::Project.fabricate_via_api! do |project|

2
qa/qa/specs/features/browser_ui/4_verify/pipeline/mr_event_rule_pipeline_spec.rb
View File

@ -5,7 +5,7 @@ module QA
context 'When job is configured to only run on merge_request_events' do
let(:mr_only_job_name) { 'mr_only_job' }
let(:non_mr_only_job_name) { 'non_mr_only_job' }
let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" }
let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" }
let(:project) do
Resource::Project.fabricate_via_api! do |project|

2
qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_child_pipeline_with_manual_spec.rb
View File

@ -3,7 +3,7 @@
module QA
RSpec.describe 'Verify', :runner, product_group: :pipeline_execution do
describe "Trigger child pipeline with 'when:manual'" do
let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" }
let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" }
let(:project) do
Resource::Project.fabricate_via_api! do |project|

2
qa/qa/specs/features/browser_ui/4_verify/pipeline/trigger_matrix_spec.rb
View File

@ -3,7 +3,7 @@
module QA
RSpec.describe 'Verify', :runner, product_group: :pipeline_authoring do
describe 'Trigger matrix' do
let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(8)}" }
let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" }
let(:project) do
Resource::Project.fabricate_via_api! do |project|

28
qa/qa/support/formatters/test_stats_formatter.rb
View File

@ -63,11 +63,11 @@ module QA
tags: {
name: example.full_description,
file_path: file_path,
status: example.execution_result.status,
status: status(example),
smoke: example.metadata.key?(:smoke).to_s,
reliable: example.metadata.key?(:reliable).to_s,
quarantined: quarantined(example.metadata),
retried: ((example.metadata[:retry_attempts] || 0) > 0).to_s,
retried: (retry_attempts(example.metadata) > 0).to_s,
job_name: job_name,
merge_request: merge_request,
run_type: run_type,
@ -81,7 +81,7 @@ module QA
api_fabrication: api_fabrication,
ui_fabrication: ui_fabrication,
total_fabrication: api_fabrication + ui_fabrication,
retry_attempts: example.metadata[:retry_attempts] || 0,
retry_attempts: retry_attempts(example.metadata),
job_url: QA::Runtime::Env.ci_job_url,
pipeline_url: env('CI_PIPELINE_URL'),
pipeline_id: env('CI_PIPELINE_ID'),
@ -158,6 +158,28 @@ module QA
(!Specs::Helpers::Quarantine.quarantined_different_context?(metadata[:quarantine])).to_s
end
# Return a more detailed status
#
# - if test is failed or pending, return rspec status
# - if test passed but had more than 1 attempt, consider test flaky
#
# @param [RSpec::Core::Example] example
# @return [String]
def status(example)
rspec_status = example.execution_result.status
return rspec_status if [:pending, :failed].include?(rspec_status)
retry_attempts(example.metadata) > 0 ? :flaky : :passed
end
# Retry attempts
#
# @param [Hash] metadata
# @return [Integer]
def retry_attempts(metadata)
metadata[:retry_attempts] || 0
end
# Print log message
#
# @param [Symbol] level

141
qa/qa/tools/delete_subgroups.rb
View File

@ -1,70 +1,149 @@
# frozen_string_literal: true
# This script deletes all subgroups of a group specified by ENV['TOP_LEVEL_GROUP_NAME']
#
# Required environment variables: GITLAB_QA_ACCESS_TOKEN and GITLAB_ADDRESS
# Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group')
# Optional environment variable: TOP_LEVEL_GROUP_NAME (defaults to 'gitlab-qa-sandbox-group-<current weekday #>')
# Optional environment variable: PERMANENTLY_DELETE (defaults to false)
# Set PERMANENTLY_DELETE to true if you would like to permanently delete subgroups on an environment with
# deletion protection enabled. Otherwise, subgroups will remain available during the retention period specified
# in admin settings. On environments with deletion protection disabled, subgroups will always be permanently deleted.
#
# Run `rake delete_subgroups`
module QA
module Tools
class DeleteSubgroups
include Support::API
include Ci::Helpers
def initialize
raise ArgumentError, "Please provide GITLAB_ADDRESS" unless ENV['GITLAB_ADDRESS']
raise ArgumentError, "Please provide GITLAB_QA_ACCESS_TOKEN" unless ENV['GITLAB_QA_ACCESS_TOKEN']
@api_client = Runtime::API::Client.new(ENV['GITLAB_ADDRESS'], personal_access_token: ENV['GITLAB_QA_ACCESS_TOKEN'])
@failed_deletion_attempts = []
end
def run
$stdout.puts 'Fetching subgroups for deletion...'
group_id = fetch_group_id
return logger.info('Top level group not found') if group_id.nil?
sub_group_ids = fetch_subgroup_ids
$stdout.puts "\nNumber of Sub Groups not already marked for deletion: #{sub_group_ids.length}"
subgroups = fetch_subgroups(group_id)
return logger.info('No subgroups available') if subgroups.empty?
delete_subgroups(sub_group_ids) unless sub_group_ids.empty?
$stdout.puts "\nDone"
subgroups_marked_for_deletion = mark_for_deletion(subgroups)
if ENV['PERMANENTLY_DELETE'] && !subgroups_marked_for_deletion.empty?
delete_permanently(subgroups_marked_for_deletion)
end
print_failed_deletion_attempts
logger.info('Done')
end
private
def delete_subgroups(sub_group_ids)
$stdout.puts "Deleting #{sub_group_ids.length} subgroups..."
sub_group_ids.each do |subgroup_id|
request_url = Runtime::API::Request.new(@api_client, "/groups/#{subgroup_id}").url
path = parse_body(get(request_url))[:full_path]
$stdout.puts "\nDeleting subgroup #{path}..."
delete_response = delete(request_url)
dot_or_f = delete_response.code == 202 ? "\e[32m.\e[0m" : "\e[31mF - #{delete_response}\e[0m"
print dot_or_f
end
end
def fetch_group_id
logger.info("Fetching top level group id...\n")
group_name = ENV['TOP_LEVEL_GROUP_NAME'] || "gitlab-qa-sandbox-group-#{Time.now.wday + 1}"
group_search_response = get Runtime::API::Request.new(@api_client, "/groups/#{group_name}" ).url
JSON.parse(group_search_response.body)["id"]
JSON.parse(group_search_response.body)['id']
end
def fetch_subgroup_ids
group_id = fetch_group_id
sub_groups_ids = []
def fetch_subgroups(group_id)
logger.info("Fetching subgroups...")
api_path = "/groups/#{group_id}/subgroups"
page_no = '1'
subgroups = []
# When we reach the last page, the x-next-page header is a blank string
while page_no.present?
$stdout.print '.'
subgroups_response = get Runtime::API::Request.new(@api_client, api_path, page: page_no, per_page: '100').url
subgroups.concat(JSON.parse(subgroups_response.body))
sub_groups_response = get Runtime::API::Request.new(@api_client, "/groups/#{group_id}/subgroups", page: page_no, per_page: '100').url
sub_groups_ids.concat(JSON.parse(sub_groups_response.body)
.reject { |subgroup| !subgroup["marked_for_deletion_on"].nil? }.map { |subgroup| subgroup['id'] })
page_no = sub_groups_response.headers[:x_next_page].to_s
page_no = subgroups_response.headers[:x_next_page].to_s
end
sub_groups_ids.uniq
subgroups
end
def subgroup_request(subgroup, **options)
Runtime::API::Request.new(@api_client, "/groups/#{subgroup['id']}", **options).url
end
def process_response_and_subgroup(response, subgroup, opts = {})
if response.code == 202
logger.info("Success\n")
opts[:save_successes_to] << subgroup if opts[:save_successes_to]
else
logger.error("Failed - #{response}\n")
@failed_deletion_attempts << { path: subgroup['full_path'], response: response }
end
end
def mark_for_deletion(subgroups)
subgroups_marked_for_deletion = []
logger.info("Marking #{subgroups.length} subgroups for deletion...\n")
subgroups.each do |subgroup|
path = subgroup['full_path']
if subgroup['marked_for_deletion_on'].nil?
logger.info("Marking subgroup #{path} for deletion...")
response = delete(subgroup_request(subgroup))
process_response_and_subgroup(response, subgroup, save_successes_to: subgroups_marked_for_deletion)
else
logger.info("Subgroup #{path} already marked for deletion\n")
subgroups_marked_for_deletion << subgroup
end
end
subgroups_marked_for_deletion
end
def subgroup_exists?(subgroup)
response = get(subgroup_request(subgroup))
if response.code == 404
logger.info("Subgroup #{subgroup['full_path']} is no longer available\n")
false
else
true
end
end
def delete_permanently(subgroups)
logger.info("Permanently deleting #{subgroups.length} subgroups...\n")
subgroups.each do |subgroup|
path = subgroup['full_path']
next unless subgroup_exists?(subgroup)
logger.info("Permanently deleting subgroup #{path}...")
delete_subgroup_response = delete(subgroup_request(subgroup, { permanently_remove: true, full_path: path }))
process_response_and_subgroup(delete_subgroup_response, subgroup)
end
end
def print_failed_deletion_attempts
if @failed_deletion_attempts.empty?
logger.info('No failed deletion attempts to report!')
else
logger.info("There were #{@failed_deletion_attempts.length} failed deletion attempts:\n")
@failed_deletion_attempts.each do |attempt|
logger.info("Subgroup: #{attempt[:path]}")
logger.error("Response: #{attempt[:response]}\n")
end
end
end
end
end

43
spec/controllers/concerns/send_file_upload_spec.rb
View File

@ -18,6 +18,12 @@ RSpec.describe SendFileUpload do
end
end
let(:cdn_uploader_class) do
Class.new(uploader_class) do
include ObjectStorage::CDN::Concern
end
end
let(:controller_class) do
Class.new do
include SendFileUpload
@ -269,5 +275,42 @@ RSpec.describe SendFileUpload do
it_behaves_like 'handles image resize requests'
end
context 'when CDN-enabled remote file is used' do
let(:uploader) { cdn_uploader_class.new(object, :file) }
let(:request) { instance_double('ActionDispatch::Request', remote_ip: '18.245.0.42') }
let(:signed_url) { 'https://cdn.example.org.test' }
let(:cdn_provider) { instance_double('ObjectStorage::CDN::GoogleCDN', signed_url: signed_url) }
before do
stub_uploads_object_storage(uploader: cdn_uploader_class)
uploader.object_store = ObjectStorage::Store::REMOTE
uploader.store!(temp_file)
allow(Gitlab.config.uploads.object_store).to receive(:proxy_download) { false }
end
context 'when use_cdn_with_job_artifacts_ui_downloads feature is enabled' do
it 'sends a file when CDN URL' do
expect(uploader).to receive(:use_cdn?).and_return(true)
expect(uploader).to receive(:cdn_provider).and_return(cdn_provider)
expect(controller).to receive(:request).and_return(request)
expect(controller).to receive(:redirect_to).with(signed_url)
subject
end
end
context 'when use_cdn_with_job_artifacts_ui_downloads is disabled' do
before do
stub_feature_flags(use_cdn_with_job_artifacts_ui_downloads: false)
end
it 'sends a file' do
expect(controller).to receive(:redirect_to).with(/#{uploader.path}/)
subject
end
end
end
end
end

43
spec/controllers/projects/artifacts_controller_spec.rb
View File

@ -153,8 +153,10 @@ RSpec.describe Projects::ArtifactsController do
end
context 'when file is stored remotely' do
let(:cdn_config) {}
before do
stub_artifacts_object_storage
stub_artifacts_object_storage(cdn: cdn_config)
create(:ci_job_artifact, :remote_store, :codequality, job: job)
end
@ -171,6 +173,45 @@ RSpec.describe Projects::ArtifactsController do
download_artifact(file_type: file_type, proxy: true)
end
end
context 'when Google CDN is configured' do
let(:cdn_config) do
{
'provider' => 'Google',
'url' => 'https://cdn.example.org',
'key_name' => 'some-key',
'key' => Base64.urlsafe_encode64(SecureRandom.hex)
}
end
before do
allow(Gitlab::ApplicationContext).to receive(:push).and_call_original
request.env['action_dispatch.remote_ip'] = '18.245.0.42'
end
context 'with use_cdn_with_job_artifacts_ui_downloads enabled' do
it 'redirects to a Google CDN request' do
expect(Gitlab::ApplicationContext).to receive(:push).with(artifact_used_cdn: true).and_call_original
download_artifact(file_type: file_type)
expect(response.redirect_url).to start_with("https://cdn.example.org/")
end
end
context 'with use_cdn_with_job_artifacts_ui_downloads disabled' do
before do
stub_feature_flags(use_cdn_with_job_artifacts_ui_downloads: false)
end
it 'does not redirect to the CDN' do
download_artifact(file_type: file_type)
expect(response.redirect_url).to be_present
expect(response.redirect_url).not_to start_with("https://cdn.example.org/")
end
end
end
end
end
end

2
spec/frontend/environments/environment_rollback_spec.js
View File

@ -76,7 +76,7 @@ describe('Rollback Component', () => {
expect(apolloProvider.defaultClient.mutate).toHaveBeenCalledWith({
mutation: setEnvironmentToRollback,
variables: { environment },
variables: { environment: { ...environment, isLastDeployment: true, retryUrl } },
});
});
});

1
spec/frontend/environments/graphql/mock_data.js
View File

@ -537,6 +537,7 @@ export const folder = {
export const resolvedEnvironment = {
id: 41,
retryUrl: '/h5bp/html5-boilerplate/-/jobs/1014/retry',
globalId: 'gid://gitlab/Environment/41',
name: 'review/hello',
state: 'available',

52
spec/lib/gitlab/database/migration_helpers_spec.rb
View File

@ -2866,58 +2866,6 @@ RSpec.describe Gitlab::Database::MigrationHelpers do
end
end
describe '#create_extension' do
subject { model.create_extension(extension) }
let(:extension) { :btree_gist }
it 'executes CREATE EXTENSION statement' do
expect(model).to receive(:execute).with(/CREATE EXTENSION IF NOT EXISTS #{extension}/)
subject
end
context 'without proper permissions' do
before do
allow(model).to receive(:execute)
.with(/CREATE EXTENSION IF NOT EXISTS #{extension}/)
.and_raise(ActiveRecord::StatementInvalid, 'InsufficientPrivilege: permission denied')
end
it 'raises an exception and prints an error message' do
expect { subject }
.to output(/user is not allowed/).to_stderr
.and raise_error(ActiveRecord::StatementInvalid, /InsufficientPrivilege/)
end
end
end
describe '#drop_extension' do
subject { model.drop_extension(extension) }
let(:extension) { 'btree_gist' }
it 'executes CREATE EXTENSION statement' do
expect(model).to receive(:execute).with(/DROP EXTENSION IF EXISTS #{extension}/)
subject
end
context 'without proper permissions' do
before do
allow(model).to receive(:execute)
.with(/DROP EXTENSION IF EXISTS #{extension}/)
.and_raise(ActiveRecord::StatementInvalid, 'InsufficientPrivilege: permission denied')
end
it 'raises an exception and prints an error message' do
expect { subject }
.to output(/user is not allowed/).to_stderr
.and raise_error(ActiveRecord::StatementInvalid, /InsufficientPrivilege/)
end
end
end
describe '#add_primary_key_using_index' do
it "executes the statement to add the primary key" do
expect(model).to receive(:execute).with /ALTER TABLE "test_table" ADD CONSTRAINT "old_name" PRIMARY KEY USING INDEX "new_name"/

65
spec/lib/gitlab/database/migrations/extension_helpers_spec.rb Normal file
View File

@ -0,0 +1,65 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Database::Migrations::ExtensionHelpers do
let(:model) do
ActiveRecord::Migration.new.extend(described_class)
end
before do
allow(model).to receive(:puts)
end
describe '#create_extension' do
subject { model.create_extension(extension) }
let(:extension) { :btree_gist }
it 'executes CREATE EXTENSION statement' do
expect(model).to receive(:execute).with(/CREATE EXTENSION IF NOT EXISTS #{extension}/)
subject
end
context 'without proper permissions' do
before do
allow(model).to receive(:execute)
.with(/CREATE EXTENSION IF NOT EXISTS #{extension}/)
.and_raise(ActiveRecord::StatementInvalid, 'InsufficientPrivilege: permission denied')
end
it 'raises an exception and prints an error message' do
expect { subject }
.to output(/user is not allowed/).to_stderr
.and raise_error(ActiveRecord::StatementInvalid, /InsufficientPrivilege/)
end
end
end
describe '#drop_extension' do
subject { model.drop_extension(extension) }
let(:extension) { 'btree_gist' }
it 'executes CREATE EXTENSION statement' do
expect(model).to receive(:execute).with(/DROP EXTENSION IF EXISTS #{extension}/)
subject
end
context 'without proper permissions' do
before do
allow(model).to receive(:execute)
.with(/DROP EXTENSION IF EXISTS #{extension}/)
.and_raise(ActiveRecord::StatementInvalid, 'InsufficientPrivilege: permission denied')
end
it 'raises an exception and prints an error message' do
expect { subject }
.to output(/user is not allowed/).to_stderr
.and raise_error(ActiveRecord::StatementInvalid, /InsufficientPrivilege/)
end
end
end
end

46
spec/uploaders/object_storage/cdn/google_cdn_spec.rb
View File

@ -92,26 +92,52 @@ RSpec.describe ObjectStorage::CDN::GoogleCDN,
end
end
describe '#signed_url' do
describe '#signed_url', :freeze_time do
let(:path) { '/path/to/file.txt' }
let(:expiration) { (Time.current + 10.minutes).utc.to_i }
let(:cdn_query_params) { "Expires=#{expiration}&KeyName=#{key_name}" }
it 'returns a valid signed URL' do
url = subject.signed_url(path)
def verify_signature(url, unsigned_url)
expect(url).to start_with("#{options[:url]}#{path}")
uri = Addressable::URI.parse(url)
parsed_query = Rack::Utils.parse_nested_query(uri.query)
signature = parsed_query.delete('Signature')
query = uri.query_values
signature = query['Signature']
signed_url = "#{options[:url]}#{path}?Expires=#{parsed_query['Expires']}&KeyName=#{key_name}"
computed_signature = OpenSSL::HMAC.digest('SHA1', key, signed_url)
computed_signature = OpenSSL::HMAC.digest('SHA1', key, unsigned_url)
aggregate_failures do
expect(parsed_query['Expires'].to_i).to be > 0
expect(parsed_query['KeyName']).to eq(key_name)
expect(query['Expires'].to_i).to be > 0
expect(query['KeyName']).to eq(key_name)
expect(signature).to eq(Base64.urlsafe_encode64(computed_signature))
end
end
context 'with default query parameters' do
let(:url) { subject.signed_url(path) }
let(:unsigned_url) { "#{options[:url]}#{path}?#{cdn_query_params}" }
it 'returns a valid signed URL' do
verify_signature(url, unsigned_url)
end
end
context 'with nil query parameters' do
let(:url) { subject.signed_url(path, params: nil) }
let(:unsigned_url) { "#{options[:url]}#{path}?#{cdn_query_params}" }
it 'returns a valid signed URL' do
verify_signature(url, unsigned_url)
end
end
context 'with extra query parameters' do
let(:url) { subject.signed_url(path, params: { 'response-content-type' => 'text/plain' }) }
let(:unsigned_url) { "#{options[:url]}#{path}?response-content-type=text%2Fplain&#{cdn_query_params}" }
it 'returns a valid signed URL' do
verify_signature(url, unsigned_url)
end
end
end
end