root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2024-09-17 09:08:25 +00:00
parent fe3ca11568
commit dac2c62de7
37 changed files with 465 additions and 277 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.rubocop_todo/layout/line_break_after_final_mixin.yml
View File

@ -125,7 +125,6 @@ Layout/LineBreakAfterFinalMixin:
- 'lib/gitlab/auth/current_user_mode.rb'
- 'lib/gitlab/auth/otp/strategies/forti_token_cloud.rb'
- 'lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb'
- 'lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb'
- 'lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb'
- 'lib/gitlab/ci/config/interpolation/config.rb'
- 'lib/gitlab/cluster/rack_timeout_observer.rb'

1
.rubocop_todo/lint/redundant_cop_disable_directive.yml
View File

@ -274,7 +274,6 @@ Lint/RedundantCopDisableDirective:
- 'lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb'
- 'lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb'
- 'lib/gitlab/background_migration/purge_stale_security_scans.rb'
- 'lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb'
- 'lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb'
- 'lib/gitlab/background_migration/update_workspaces_config_version.rb'
- 'lib/gitlab/background_migration/update_workspaces_config_version3.rb'

1
.rubocop_todo/migration/background_migration_record.yml
View File

@ -17,5 +17,4 @@ Migration/BackgroundMigrationRecord:
- 'lib/gitlab/background_migration/populate_latest_pipeline_ids.rb'
- 'lib/gitlab/background_migration/project_namespaces/models/namespace.rb'
- 'lib/gitlab/background_migration/project_namespaces/models/project.rb'
- 'lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb'
- 'lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb'

1
.rubocop_todo/style/class_and_module_children.yml
View File

@ -472,7 +472,6 @@ Style/ClassAndModuleChildren:
- 'ee/lib/gitlab/path_locks_finder.rb'
- 'lib/api/error_tracking/client_keys.rb'
- 'lib/api/error_tracking/project_settings.rb'
- 'lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb'
- 'lib/gitlab/ci/badge/base.rb'
- 'lib/gitlab/ci/badge/coverage/metadata.rb'
- 'lib/gitlab/ci/badge/coverage/report.rb'

1
.rubocop_todo/style/inline_disable_annotation.yml
View File

@ -2081,7 +2081,6 @@ Style/InlineDisableAnnotation:
- 'lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb'
- 'lib/gitlab/background_migration/purge_stale_security_scans.rb'
- 'lib/gitlab/background_migration/redis/backfill_project_pipeline_status_ttl.rb'
- 'lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb'
- 'lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb'
- 'lib/gitlab/background_migration/update_workspaces_config_version.rb'
- 'lib/gitlab/background_task.rb'

2
GITALY_SERVER_VERSION
View File

@ -1 +1 @@
af567077c99010d59d5e44b5307a3fdad12c886e
5b34830d209669e3ffa45b917104474435e0bd90

2
GITLAB_KAS_VERSION
View File

@ -1 +1 @@
c8cfabff3dc8dbec999fda740e85a0bbf07789a0
ca089de1bada3b8c5470c7b0a5356b02e33a24e8

20
app/models/ci/build_metadata.rb
View File

@ -73,6 +73,19 @@ module Ci
true
end
def manual_confirmation_message
return unless config_options && config_options[:manual_confirmation]
variables = config_variables_to_hash
config_options[:manual_confirmation].gsub(/\$[A-Za-z]\w*/) do |match|
name = match.delete_prefix('$')
value = variables[name]
next match unless value
"$#{name}=#{value}"
end
end
private
def set_build_project
@ -110,5 +123,12 @@ module Ci
def runner_timeout_set?
build.runner&.maximum_timeout.to_i > 0
end
def config_variables_to_hash
config_variables.each_with_object({}) do |item, hash|
hash[item[:key]] = item[:value]
hash
end
end
end
end

2
app/models/ci/processable.rb
View File

@ -256,7 +256,7 @@ module Ci
end
def manual_confirmation_message
options[:manual_confirmation] if manual_job?
metadata.manual_confirmation_message if manual_job?
end
private

4
doc/administration/geo/replication/configuration.md
View File

@ -225,7 +225,7 @@ In the following steps, replace `<ssh_host_key_path>` with the one you're using:
1. On the left sidebar, at the bottom, select **Admin**.
1. On the left sidebar, select **Geo > Sites**.
1. Select **Add site**.
![Add secondary site](img/adding_a_secondary_v15_8.png)
![Adding a secondary site in Geo configuration interface](img/adding_a_secondary_v15_8.png)
1. In **Name**, enter the value for `gitlab_rails['geo_node_name']` in
`/etc/gitlab/gitlab.rb`. These values must always match **exactly**, character
for character.
@ -355,7 +355,7 @@ The initial replication may take some time. The status of the site or the 'backf
can monitor the synchronization process on each Geo site from the **primary**
site's **Geo Sites** dashboard in your browser.
![Geo dashboard](img/geo_dashboard_v14_0.png)
![Geo dashboard of secondary site](img/geo_dashboard_v14_0.png)
If your installation isn't working properly, check the
[troubleshooting document](troubleshooting/index.md).

16
doc/administration/geo/replication/location_aware_git_url.md
View File

@ -57,20 +57,20 @@ routing configurations.
[Route53 dashboard](https://console.aws.amazon.com/route53/home) and select
**Traffic policies**.
![Traffic policies](img/single_git_traffic_policies.png)
![Traffic policies section of Route53 dashboard](img/single_git_traffic_policies.png)
1. Select **Create traffic policy**.
![Name policy](img/single_git_name_policy.png)
![Naming the traffic policy](img/single_git_name_policy.png)
1. Fill in the **Policy Name** field with `Single Git Host` and select **Next**.
![Policy diagram](img/single_git_policy_diagram.png)
![Selecting DNS type for traffic policy](img/single_git_policy_diagram.png)
1. Leave **DNS type** as `A: IP Address in IPv4 format`.
1. Select **Connect to** and select **Geolocation rule**.
![Add geolocation rule](img/single_git_add_geolocation_rule.png)
![Adding the geolocation rule](img/single_git_add_geolocation_rule.png)
1. For the first **Location**, leave it as `Default`.
1. Select **Connect to** and select **New endpoint**.
@ -79,16 +79,16 @@ routing configurations.
1. Select **Connect to** and select **New endpoint**.
1. Choose **Type** `value` and fill it in with `<your **secondary** IP address>`.
![Add traffic policy endpoints](img/single_git_add_traffic_policy_endpoints.png)
![Setting locations and endpoints to geolocation rule](img/single_git_add_traffic_policy_endpoints.png)
1. Select **Create traffic policy**.
![Create policy records with traffic policy](img/single_git_create_policy_records_with_traffic_policy.png)
![Setting up policy records in traffic policy](img/single_git_create_policy_records_with_traffic_policy.png)
1. Fill in **Policy record DNS name** with `git`.
1. Select **Create policy records**.
![Created policy record](img/single_git_created_policy_record.png)
![Successfully created traffic policy with policy records](img/single_git_created_policy_record.png)
You have successfully set up a single host, for example, `git.example.com` which
distributes traffic to your Geo sites by geolocation!
@ -102,7 +102,7 @@ on the external URL of the current host. For example:
- `git@secondary.example.com:group1/project1.git`
- `https://secondary.example.com/group1/project1.git`
![Clone panel](img/single_git_clone_panel.png)
![SSH and HTTPS URLs of repository](img/single_git_clone_panel.png)
You can customize the:

45
doc/administration/settings/sign_up_restrictions.md
View File

@ -136,6 +136,51 @@ To remove the user cap:
1. Remove the number from **User cap**.
1. Select **Save changes**.
## Turn on restricted access
DETAILS:
**Tier:** Premium, Ultimate
**Offering:** GitLab.com
**Status:** Beta
> -Introduced in GitLab 17.5.
This feature is in [beta](../../policy/experiment-beta-support.md).
Use restricted access to prevent overage fees. Overage fees occur when you exceed the number of seats
in your subscription, and must be paid at the next [quarterly reconciliation](../../subscriptions/quarterly_reconciliation.md).
When you turn on restricted access, groups cannot add new billable users when there are no seats
left in the subscription.
Prerequisites:
- You must have the Owner role for the group.
- The group or one of its subgroups or projects must not be shared externally.
To turn on restricted access:
1. On the left sidebar, select **Settings > General**.
1. Expand **Permissions and group features**.
1. Under **Seat controls**, select **Restricted access**.
### Known issues
When you turn on restricted access, the following known issues might occur and result in overages:
- The number of seats can still be exceeded if:
- Multiple users with the Owner role add members simultaneously.
- New billable members delay accepting an invitation.
- You change from using the user cap to restricted access, and have members pending approval
from before you changed to restricted access, those members remain in a pending state. If
pending members are approved while using restricted access, you might exceed the number of seats in your subscription.
- If you renew your subscription through the GitLab Sales Team for less users than your current
subscription, you will incur an overage fee. To avoid this fee, remove additional users before your
renewal starts. For example:
- You have 20 users.
- You renew your subscription for 15 users.
- You will be charged overages for the five additional users.
## Minimum password length limit
You can [change](../../security/password_length_limits.md#modify-minimum-password-length)

2
doc/api/group_ssh_certificates.md
View File

@ -23,7 +23,7 @@ To use this API, you must [authenticate yourself](rest/index.md#authentication)
## Get all SSH certificates for a particular group
```plaintext
GET groups/:id/ssh_certificates
GET /groups/:id/ssh_certificates
```
Parameters:

4
doc/ci/runners/new_creation_workflow.md
View File

@ -84,6 +84,10 @@ To continue using registration tokens after GitLab 17.0:
Existing runners will continue to work as usual even after 18.0. This change only affects registration of new runners.
The [GitLab Runner Helm chart](https://docs.gitlab.com/runner/install/kubernetes.html) generates new runner pods every time a job is executed.
For these runners, [enable legacy runner registration](#using-registration-tokens-after-gitlab-170) to use registration tokens.
In GitLab 18.0 and later, you must migrate to the [new runner registration workflow](#the-new-runner-registration-workflow).
## Changes to the `gitlab-runner register` command syntax
The `gitlab-runner register` command will stop accepting registration tokens and instead accept new runner

3
doc/development/i18n/proofreader.md
View File

@ -142,7 +142,8 @@ translations to the GitLab project.
In the merge request description, include links to any projects you have previously translated.
1. [GitLab team members](https://about.gitlab.com/company/team/),
[Core team members](https://about.gitlab.com/community/core-team/),
[core team members](https://about.gitlab.com/community/core-team/),
[globalization and localization team members](https://handbook.gitlab.com/handbook/marketing/localization/),
or current proofreaders fluent in the language consider your request to become a proofreader
based on the merits of your previous translations.

4
doc/subscriptions/gitlab_com/index.md
View File

@ -186,6 +186,10 @@ For example, if you purchase a subscription for 10 users:
Seats owed = 12 - 10 (Maximum users - users in subscription)
To prevent charges from seats owed, you can
[turn on restricted access](../../administration/settings/sign_up_restrictions.md#turn-on-restricted-access).
This setting restricts groups from adding new billable users when there are no seats left in the subscription.
### Free Guest users
DETAILS:

3
doc/subscriptions/quarterly_reconciliation.md
View File

@ -15,6 +15,9 @@ In accordance with [the GitLab Subscription Agreement](https://about.gitlab.com/
GitLab reviews your seat usage and sends you an invoice for any overages.
This review occurs either quarterly (quarterly reconciliation process) or annually (annual true-up process).
To prevent overages, you can [turn on restricted access](../administration/settings/sign_up_restrictions.md#turn-on-restricted-access).
This setting restricts groups from adding new billable users when there are no seats left in the subscription.
## Quarterly reconciliation versus annual true-ups
With **quarterly reconciliation**, you are billed per quarter on a prorated basis for the remaining portion of the subscription term.

2
doc/user/gitlab_com/index.md
View File

@ -406,7 +406,7 @@ GitLab can rate-limit requests at several layers. The rate limits listed here
are configured in the application. These limits are the most
restrictive per IP address. For more information about the rate limits
for GitLab.com, see
[an overview](https://gitlab.com/gitlab-com/runbooks/-/tree/master/docs/rate-limiting).
[the documentation in the handbook](https://handbook.gitlab.com/handbook/engineering/infrastructure/rate-limiting).
### Rate limiting responses

2
doc/user/profile/achievements.md
View File

@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
DETAILS:
**Tier:** Free, Premium, Ultimate
**Offering:** GitLab.com, Self-managed, GitLab Dedicated
**Offering:** GitLab.com, Self-managed
**Status:** Experiment
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/113156) in GitLab 15.10 [with a flag](../../administration/feature_flags.md) named `achievements`. Disabled by default.

4
doc/user/project/protected_tags.md
View File

@ -27,8 +27,8 @@ This feature evolved out of [protected branches](repository/branches/protected.m
By default:
- To create tags, you must have the Maintainer role.
- No one can update or delete tags.
- To create or delete tags, you must have the Maintainer role.
- Protected tags [can only be deleted](#delete-a-protected-tag) using the UI or API.
## Configuring protected tags

26
doc/user/project/web_ide/index.md
View File

@ -285,6 +285,32 @@ As a workaround:
For more information, see [VS Code issue 80170](https://github.com/microsoft/vscode/issues/80170).
### Update the OAuth callback URL
DETAILS:
**Offering:** Self-managed
Prerequisites:
- You must have administrator access to the instance.
The Web IDE uses an [instance-wide OAuth application](../../../integration/oauth_provider.md#create-an-instance-wide-application) for authentication.
If the OAuth callback URL is misconfigured, you might encounter a `Cannot open Web IDE` error page with the following message:
```plaintext
The URL you're using to access the Web IDE and the configured OAuth callback URL do not match. This issue often occurs when you're using a proxy.
```
To resolve this issue, you must update the OAuth callback URL to match the URL used to access the GitLab instance.
To update the OAuth callback URL:
1. On the left sidebar, at the bottom, select **Admin**.
1. Select **Applications**.
1. For **GitLab Web IDE**, select **Edit**.
1. Enter the OAuth callback URL.
You can enter multiple URLs separated by newlines.
### Report a problem
To report a problem, [create a new issue](https://gitlab.com/gitlab-org/gitlab-web-ide/-/issues/new)

59
lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb
View File

@ -1,59 +0,0 @@
# frozen_string_literal: true
# This migration will look for Vulnerabilities::Finding objects that would have a duplicate UUIDv5 if the UUID was
# recalculated. Then it removes Vulnerabilities::FindingPipeline objects associated with those Findings.
# We can't just drop those Findings directly since the cascade drop will timeout if any given Finding has too many
# associated FindingPipelines
class Gitlab::BackgroundMigration::RemoveOccurrencePipelinesAndDuplicateVulnerabilitiesFindings
# rubocop:disable Gitlab/NamespacedClass, Style/Documentation
class VulnerabilitiesFinding < ActiveRecord::Base
self.table_name = "vulnerability_occurrences"
end
class VulnerabilitiesFindingPipeline < ActiveRecord::Base
include EachBatch
self.table_name = "vulnerability_occurrence_pipelines"
end
# rubocop:enable Gitlab/NamespacedClass, Style/Documentation
def perform(start_id, end_id)
ids_to_look_for = findings_that_would_produce_duplicate_uuids(start_id, end_id)
ids_to_look_for.each do |finding_id|
VulnerabilitiesFindingPipeline.where(occurrence_id: finding_id).each_batch(of: 1000) do |pipelines|
pipelines.delete_all
end
end
VulnerabilitiesFinding.where(id: ids_to_look_for).delete_all
mark_job_as_succeeded(start_id, end_id)
end
private
def findings_that_would_produce_duplicate_uuids(start_id, end_id)
VulnerabilitiesFinding
.from("vulnerability_occurrences to_delete")
.where("to_delete.id BETWEEN ? AND ?", start_id, end_id)
.where(
"EXISTS (
SELECT 1
FROM vulnerability_occurrences duplicates
WHERE duplicates.report_type = to_delete.report_type
AND duplicates.location_fingerprint = to_delete.location_fingerprint
AND duplicates.primary_identifier_id = to_delete.primary_identifier_id
AND duplicates.project_id = to_delete.project_id
AND duplicates.id > to_delete.id
)"
)
.pluck("to_delete.id")
end
def mark_job_as_succeeded(*arguments)
Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
self.class.name.demodulize,
arguments
)
end
end

43
lib/gitlab/database/background_migration/batched_background_migration_dictionary.rb
View File

@ -1,43 +0,0 @@
# frozen_string_literal: true
module Gitlab
module Database
module BackgroundMigration
class BatchedBackgroundMigrationDictionary
def self.entry(migration_job_name)
entries_by_migration_job_name[migration_job_name]
end
private_class_method def self.entries_by_migration_job_name
@entries_by_migration_job_name ||= Dir.glob(dict_path).to_h do |file_path|
entry = Entry.new(file_path)
[entry.migration_job_name, entry]
end
end
private_class_method def self.dict_path
Rails.root.join('db/docs/batched_background_migrations/*.yml')
end
class Entry
def initialize(file_path)
@file_path = file_path
@data = YAML.load_file(file_path)
end
def migration_job_name
data['migration_job_name']
end
def finalized_by
data['finalized_by']
end
private
attr_reader :file_path, :data
end
end
end
end
end

36
lib/gitlab/database/migrations/batched_background_migration_helpers.rb
View File

@ -19,6 +19,14 @@ module Gitlab
BATCH_MIN_VALUE = 1 # Default minimum value for batched migrations
BATCH_MIN_DELAY = 2.minutes.freeze # Minimum delay between batched migrations
ENFORCE_EARLY_FINALIZATION_FROM_VERSION = '20240905124117'
EARLY_FINALIZATION_ERROR = <<-MESSAGE.squeeze(' ').strip
Batched migration should be finalized only after at-least one required stop from queuing it.
This is to ensure that we are not breaking the upgrades for self-managed instances.
For more info visit: https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#finalize-a-batched-background-migration
MESSAGE
# Creates a batched background migration for the given table. A batched migration runs one job
# at a time, computing the bounds of the next batch based on the current migration settings and the previous
# batch bounds. Each job's execution status is tracked in the database as the migration runs. The given job
@ -188,7 +196,14 @@ module Gitlab
end
end
def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:, finalize: true)
def ensure_batched_background_migration_is_finished(
job_class_name:,
table_name:,
column_name:,
job_arguments:,
finalize: true,
skip_early_finalization_validation: false
)
Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode!
if transaction_open?
@ -216,6 +231,10 @@ module Gitlab
return Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" if migration.nil?
if migration.respond_to?(:queued_migration_version) && !skip_early_finalization_validation
prevent_early_finalization!(migration.queued_migration_version, version)
end
return if migration.finalized?
if migration.finished?
@ -269,6 +288,21 @@ module Gitlab
end
# rubocop:enable GitlabSecurity/PublicSend
end
def prevent_early_finalization!(queued_migration_version, version)
return if version.to_s <= ENFORCE_EARLY_FINALIZATION_FROM_VERSION || queued_migration_version.blank?
queued_migration_milestone = Gitlab::Utils::BatchedBackgroundMigrationsDictionary
.new(queued_migration_version)
.milestone
return unless queued_migration_milestone.present?
queued_migration_milestone = Gitlab::VersionInfo.parse_from_milestone(queued_migration_milestone)
last_required_stop = Gitlab::Database.upgrade_path.last_required_stop
raise EARLY_FINALIZATION_ERROR unless queued_migration_milestone <= last_required_stop
end
end
end
end

74
lib/gitlab/utils/batched_background_migrations_dictionary.rb Normal file
View File

@ -0,0 +1,74 @@
# frozen_string_literal: true
module Gitlab
module Utils
class BatchedBackgroundMigrationsDictionary
DICTIONARY_BASE_DIR = 'db/docs/batched_background_migrations'
attr_reader :queued_migration_version
class << self
def entries
return @entries if @entries.present? && !Rails.env.test?
@entries = Dir.glob("*.yml", base: DICTIONARY_BASE_DIR).each_with_object({}) do |file_name, data|
dictionary = YAML.load_file(File.join(DICTIONARY_BASE_DIR, file_name))
next unless dictionary['queued_migration_version'].present?
data[dictionary['queued_migration_version'].to_s] = {
migration_job_name: dictionary['migration_job_name'],
introduced_by_url: dictionary['introduced_by_url'],
finalized_by: dictionary['finalized_by'].to_s,
milestone: dictionary['milestone']
}
data[dictionary['migration_job_name']] = data[dictionary['queued_migration_version'].to_s].merge(
queued_migration_version: dictionary['queued_migration_version']
)
end
end
def entry(migration_job_name)
return unless entries&.dig(migration_job_name)
new(entries[migration_job_name][:queued_migration_version])
end
# Used by BackgroundMigration/DictionaryFile cop to invalidate its cache
# if the contents of `db/docs/batched_background_migrations` changes.
def checksum(skip_memoization: false)
return @checksum if @checksum.present? && !skip_memoization
@checksum = Digest::SHA256.hexdigest(entries.to_s)
end
end
def initialize(queued_migration_version)
@queued_migration_version = queued_migration_version
end
def finalized_by
entry&.dig(:finalized_by)
end
def introduced_by_url
entry&.dig(:introduced_by_url)
end
def milestone
entry&.dig(:milestone)
end
def migration_job_name
entry&.dig(:migration_job_name)
end
private
def entry
@entry ||= self.class.entries[queued_migration_version.to_s]
end
end
end
end

51
rubocop/batched_background_migrations_dictionary.rb
View File

@ -1,51 +0,0 @@
# frozen_string_literal: true
module RuboCop
class BatchedBackgroundMigrationsDictionary
DICTIONARY_BASE_DIR = 'db/docs/batched_background_migrations'
attr_reader :queued_migration_version
class << self
def dictionary_data
@dictionary_data ||= Dir.glob("*.yml", base: DICTIONARY_BASE_DIR).each_with_object({}) do |file_name, data|
dictionary = YAML.load_file(File.join(DICTIONARY_BASE_DIR, file_name))
next unless dictionary['queued_migration_version'].present?
data[dictionary['queued_migration_version'].to_s] = {
introduced_by_url: dictionary['introduced_by_url'],
finalized_by: dictionary['finalized_by'].to_s,
milestone: dictionary['milestone']
}
end
end
def checksum
@checksum ||= Digest::SHA256.hexdigest(dictionary_data.to_s)
end
end
def initialize(queued_migration_version)
@queued_migration_version = queued_migration_version
end
def finalized_by
dictionary_data&.dig(:finalized_by)
end
def introduced_by_url
dictionary_data&.dig(:introduced_by_url)
end
def milestone
dictionary_data&.dig(:milestone)
end
private
def dictionary_data
@dictionary_data ||= self.class.dictionary_data[queued_migration_version.to_s]
end
end
end

6
rubocop/cop/background_migration/dictionary_file.rb
View File

@ -1,7 +1,7 @@
# frozen_string_literal: true
require_relative '../../migration_helpers'
require_relative '../../batched_background_migrations_dictionary'
require_relative '../../../lib/gitlab/utils/batched_background_migrations_dictionary'
URL_PATTERN = %r{\Ahttps://gitlab\.com/gitlab-org/gitlab/-/merge_requests/\d+\z}
@ -52,7 +52,7 @@ module RuboCop
end
def external_dependency_checksum
RuboCop::BatchedBackgroundMigrationsDictionary.checksum
::Gitlab::Utils::BatchedBackgroundMigrationsDictionary.checksum
end
private
@ -78,7 +78,7 @@ module RuboCop
return [:missing_dictionary, { file_name: dictionary_file_path(migration_name) }]
end
bbm_dictionary = RuboCop::BatchedBackgroundMigrationsDictionary.new(version(node))
bbm_dictionary = ::Gitlab::Utils::BatchedBackgroundMigrationsDictionary.new(version(node))
return [:missing_key, { key: :milestone }] unless bbm_dictionary.milestone.present?

6
rubocop/cop/migration/unfinished_dependencies.rb
View File

@ -1,7 +1,7 @@
# frozen_string_literal: true
require_relative '../../migration_helpers'
require_relative '../../batched_background_migrations_dictionary'
require_relative '../../../lib/gitlab/utils/batched_background_migrations_dictionary'
module RuboCop
module Cop
@ -43,7 +43,9 @@ module RuboCop
private
def fetch_finalized_by(queued_migration_version)
BatchedBackgroundMigrationsDictionary.new(queued_migration_version).finalized_by
::Gitlab::Utils::BatchedBackgroundMigrationsDictionary
.new(queued_migration_version)
.finalized_by
end
end
end

13
spec/lib/gitlab/database/background_migration/batched_background_migration_dictionary_spec.rb
View File

@ -1,13 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe ::Gitlab::Database::BackgroundMigration::BatchedBackgroundMigrationDictionary, feature_category: :database do
describe '.entry' do
it 'returns a single dictionary entry for the given migration job' do
entry = described_class.entry('MigrateHumanUserType')
expect(entry.migration_job_name).to eq('MigrateHumanUserType')
expect(entry.finalized_by).to eq(20230523101514)
end
end
end

90
spec/lib/gitlab/database/migrations/batched_background_migration_helpers_spec.rb
View File

@ -443,6 +443,18 @@ RSpec.describe Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers,
end
end
shared_examples 'invalid early finalization' do
it 'throws an early finalization error' do
expect { ensure_batched_background_migration_is_finished }.to raise_error(described_class::EARLY_FINALIZATION_ERROR)
end
end
shared_examples 'valid finalization' do
it 'does not throw any error' do
expect { ensure_batched_background_migration_is_finished }.not_to raise_error
end
end
describe '#ensure_batched_background_migration_is_finished' do
let(:job_class_name) { 'CopyColumnUsingBackgroundMigrationJob' }
let(:table_name) { '_test_table' }
@ -461,12 +473,14 @@ RSpec.describe Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers,
let(:migration_attributes) do
configuration.merge(
gitlab_schema: gitlab_schema
gitlab_schema: gitlab_schema,
queued_migration_version: Time.now.utc.strftime("%Y%m%d%H%M%S")
)
end
before do
allow(migration).to receive(:transaction_open?).and_return(false)
allow(migration).to receive(:version).and_return('20240905124118')
end
subject(:ensure_batched_background_migration_is_finished) { migration.ensure_batched_background_migration_is_finished(**configuration) }
@ -588,5 +602,79 @@ RSpec.describe Gitlab::Database::Migrations::BatchedBackgroundMigrationHelpers,
expect { migration.ensure_batched_background_migration_is_finished(**configuration.merge(finalize: false)) }.to raise_error(RuntimeError)
end
end
context 'with finalized migration' do
let(:migration_attributes) do
configuration
.except(:skip_early_finalization_validation)
.merge(queued_migration_version: '20240905124118')
end
before do
allow(Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas).to receive(:require_dml_mode!)
create(:batched_background_migration, :finalized, migration_attributes)
end
context 'when the migration does not have queued_migration_version attr' do
let(:migration_attributes) { configuration.merge(queued_migration_version: nil) }
it_behaves_like 'valid finalization'
end
context 'when the migration version is before ENFORCE_EARLY_FINALIZATION_FROM_VERSION' do
let(:migration_attributes) { configuration.merge(queued_migration_version: '20240905124116') }
it_behaves_like 'valid finalization'
end
context 'when the migration is queued after the last required stop' do
before do
stub_bbm_stops('16.11', '17.2')
end
it_behaves_like 'invalid early finalization'
context 'with skip_early_finalization_validation enabled' do
let(:configuration) do
{
job_class_name: job_class_name,
table_name: table_name,
column_name: column_name,
job_arguments: job_arguments,
skip_early_finalization_validation: true
}
end
it_behaves_like 'valid finalization'
end
end
context 'when the migration is queued on the last required stop' do
before do
stub_bbm_stops('16.11', '16.11')
end
it_behaves_like 'valid finalization'
end
context 'when the migration is queued before the last required stop' do
before do
stub_bbm_stops('16.11', '16.10')
end
it_behaves_like 'valid finalization'
end
end
end
def stub_bbm_stops(last_required_stop, queued_milestone)
allow_next_instance_of(Gitlab::Utils::BatchedBackgroundMigrationsDictionary) do |dict|
allow(dict).to receive(:milestone).and_return(queued_milestone)
end
allow_next_instance_of(Gitlab::Utils::UpgradePath) do |ugrade_path|
allow(ugrade_path).to receive(:last_required_stop).and_return(Gitlab::VersionInfo.parse_from_milestone(last_required_stop))
end
end
end

75
spec/lib/gitlab/utils/batched_background_migrations_dictionary_spec.rb Normal file
View File

@ -0,0 +1,75 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Utils::BatchedBackgroundMigrationsDictionary, feature_category: :database do
let(:bbm_dictionary_file_name) { "#{described_class::DICTIONARY_BASE_DIR}/test_migration.yml" }
let(:migration_version) { 20230307160250 }
let(:finalized_by) { '20230307160255' }
let(:introduced_by_url) { 'https://test_url' }
let(:milestone) { '16.5' }
let(:migration_job_name) { 'TestMigration' }
let(:bbm_dictionary_data) do
{
migration_job_name: migration_job_name,
feature_category: :database,
introduced_by_url: introduced_by_url,
milestone: milestone,
queued_migration_version: migration_version,
finalized_by: finalized_by
}
end
before do
File.open(bbm_dictionary_file_name, 'w') do |file|
file.write(bbm_dictionary_data.stringify_keys.to_yaml)
end
end
after do
FileUtils.rm(bbm_dictionary_file_name)
end
subject(:batched_background_migration) { described_class.new(migration_version) }
describe '.entry' do
it 'returns a single dictionary entry for the given migration job' do
entry = described_class.entry('TestMigration')
expect(entry.migration_job_name).to eq('TestMigration')
expect(entry.finalized_by.to_s).to eq(finalized_by)
end
end
shared_examples 'safely returns bbm attribute' do |attribute|
it 'returns the attr of the bbm' do
expect(batched_background_migration.public_send(attribute)).to eq(public_send(attribute))
end
it 'returns nothing for non-existing bbm dictionary' do
expect(described_class.new('random').public_send(attribute)).to be_nil
end
end
describe '#introduced_by_url' do
it_behaves_like 'safely returns bbm attribute', :introduced_by_url
end
describe '#milestone' do
it_behaves_like 'safely returns bbm attribute', :milestone
end
describe '#migration_job_name' do
it_behaves_like 'safely returns bbm attribute', :migration_job_name
end
describe '.checksum' do
let(:entries) { { c: "d", a: "b" } }
it 'returns a checksum of the entries' do
allow(described_class).to receive(:entries).and_return(entries)
expect(described_class.checksum(skip_memoization: true)).to eq(Digest::SHA256.hexdigest(entries.to_s))
end
end
end

48
spec/models/ci/build_metadata_spec.rb
View File

@ -267,4 +267,52 @@ RSpec.describe Ci::BuildMetadata, feature_category: :continuous_integration do
expect { metadata.config_options }.not_to change(metadata, :changes)
end
end
describe '#manual_confirmation_message' do
context 'with user defined variables' do
describe 'variable matched' do
it 'return message with substituted variables matched' do
metadata.config_options = { manual_confirmation: "Deploy to $ENVIRONMENT_NAME and region $REGION_NAME" }
metadata.config_variables = [{ key: 'ENVIRONMENT_NAME', value: 'production' },
{ key: 'REGION_NAME', value: 'us-east' }]
expect(metadata.manual_confirmation_message)
.to eq('Deploy to $ENVIRONMENT_NAME=production and region $REGION_NAME=us-east')
end
it 'escaped single quotes removal' do
metadata.config_options = { manual_confirmation: "Deploy to $ENVIRONMENT. Confirm it\'s done" }
metadata.config_variables = [{ key: 'ENVIRONMENT', value: 'production' }]
expect(metadata.manual_confirmation_message).to eq("Deploy to $ENVIRONMENT=production. Confirm it's done")
end
end
context 'variable not matched' do
it 'return original message' do
metadata.config_options = { manual_confirmation: "Deploy to $ENVIRONMENT_NAME" }
metadata.config_variables = [{ key: 'REGION_NAME', value: 'us-east' }]
expect(metadata.manual_confirmation_message).to eq("Deploy to $ENVIRONMENT_NAME")
end
it 'return original message if partial matched' do
metadata.config_options = { manual_confirmation: "Deploy to $ENVIRONMENT_NAME_REAL" }
metadata.config_variables = [{ key: '$ENVIRONMENT_NAME', value: 'Production' }]
expect(metadata.manual_confirmation_message).to eq("Deploy to $ENVIRONMENT_NAME_REAL")
end
end
end
context 'without user defined variables' do
it 'return original message' do
metadata.config_options = { manual_confirmation: "Deploy to production" }
expect(metadata.manual_confirmation_message).to eq('Deploy to production')
end
end
context 'manual_confirmation is nil' do
it 'return nil' do
metadata.config_options = {}
expect(metadata.manual_confirmation_message).to be_nil
end
end
end
end

67
spec/rubocop/batched_background_migrations_dictionary_spec.rb
View File

@ -1,67 +0,0 @@
# frozen_string_literal: true
require 'rubocop_spec_helper'
require_relative '../../rubocop/batched_background_migrations_dictionary'
RSpec.describe RuboCop::BatchedBackgroundMigrationsDictionary, feature_category: :database do
let(:bbm_dictionary_file_name) { "#{described_class::DICTIONARY_BASE_DIR}/test_migration.yml" }
let(:migration_version) { 20230307160250 }
let(:finalized_by_version) { 20230307160255 }
let(:introduced_by_url) { 'https://test_url' }
let(:bbm_dictionary_data) do
{
migration_job_name: 'TestMigration',
feature_category: :database,
introduced_by_url: introduced_by_url,
milestone: 16.5,
queued_migration_version: migration_version,
finalized_by: finalized_by_version
}
end
before do
File.open(bbm_dictionary_file_name, 'w') do |file|
file.write(bbm_dictionary_data.stringify_keys.to_yaml)
end
end
after do
FileUtils.rm(bbm_dictionary_file_name)
end
subject(:batched_background_migration) { described_class.new(migration_version) }
describe '#finalized_by' do
it 'returns the finalized_by version of the bbm with given version',
quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/456913' do
expect(batched_background_migration.finalized_by).to eq(finalized_by_version.to_s)
end
it 'returns nothing for non-existing bbm dictionary' do
expect(described_class.new('random').finalized_by).to be_nil
end
end
describe '#introduced_by_url' do
it 'returns the introduced_by_url of the bbm with given version',
quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/456912' do
expect(batched_background_migration.introduced_by_url).to eq(introduced_by_url)
end
it 'returns nothing for non-existing bbm dictionary' do
expect(described_class.new('random').introduced_by_url).to be_nil
end
end
describe '.checksum' do
let(:dictionary_data) { { c: "d", a: "b" } }
it 'returns a checksum of the dictionary_data' do
allow(described_class).to receive(:dictionary_data).and_return(dictionary_data)
expect(described_class.checksum).to eq(Digest::SHA256.hexdigest(dictionary_data.to_s))
end
end
end

18
spec/rubocop/cop/background_migration/dictionary_file_spec.rb
View File

@ -140,12 +140,13 @@ RSpec.describe RuboCop::Cop::BackgroundMigration::DictionaryFile, feature_catego
before do
allow(File).to receive(:exist?).and_call_original
allow(File).to receive(:exist?).with(dictionary_file_path).and_return(true)
allow(::RuboCop::BatchedBackgroundMigrationsDictionary).to receive(:dictionary_data).and_return({
'20231118100907' => {
introduced_by_url: introduced_by_url,
milestone: milestone
}
})
allow(Gitlab::Utils::BatchedBackgroundMigrationsDictionary)
.to receive(:entries).and_return({
'20231118100907' => {
introduced_by_url: introduced_by_url,
milestone: milestone
}
})
end
context 'without introduced_by_url' do
@ -221,8 +222,9 @@ RSpec.describe RuboCop::Cop::BackgroundMigration::DictionaryFile, feature_catego
end
describe '#external_dependency_checksum' do
it 'uses the RuboCop::BatchedBackgroundMigrationsDictionary.checksum' do
allow(RuboCop::BatchedBackgroundMigrationsDictionary).to receive(:checksum).and_return('aaaaa')
it 'uses the Utils::BatchedBackgroundMigrationsDictionary.checksum' do
allow(Gitlab::Utils::BatchedBackgroundMigrationsDictionary)
.to receive(:checksum).and_return('aaaaa')
expect(cop.external_dependency_checksum).to eq('aaaaa')
end

2
spec/rubocop/cop/migration/unfinished_dependencies_spec.rb
View File

@ -99,7 +99,7 @@ RSpec.describe RuboCop::Cop::Migration::UnfinishedDependencies, feature_category
context 'with properly finalized dependent background migrations' do
before do
allow_next_instance_of(RuboCop::BatchedBackgroundMigrationsDictionary) do |bbms|
allow_next_instance_of(Gitlab::Utils::BatchedBackgroundMigrationsDictionary) do |bbms|
allow(bbms).to receive(:finalized_by).and_return(version - 5)
end
end

4
spec/support/helpers/migrations_helpers.rb
View File

@ -141,10 +141,10 @@ module MigrationsHelpers
end
def finalized_by_version
finalized_by = ::Gitlab::Database::BackgroundMigration::BatchedBackgroundMigrationDictionary
finalized_by = ::Gitlab::Utils::BatchedBackgroundMigrationsDictionary
.entry(described_class.to_s.demodulize)&.finalized_by
finalized_by.to_i if finalized_by
finalized_by.to_i if finalized_by.present?
end
def migration_schema_version

4
spec/support_specs/helpers/migrations_helpers_spec.rb
View File

@ -113,7 +113,7 @@ RSpec.describe MigrationsHelpers, feature_category: :database do
before do
allow(helper).to receive(:described_class)
allow(::Gitlab::Database::BackgroundMigration::BatchedBackgroundMigrationDictionary).to(
allow(::Gitlab::Utils::BatchedBackgroundMigrationsDictionary).to(
receive(:entry).and_return(dictionary_entry)
)
end
@ -125,7 +125,7 @@ RSpec.describe MigrationsHelpers, feature_category: :database do
context 'when finalized_by is a string' do
let(:dictionary_entry) do
instance_double(
::Gitlab::Database::BackgroundMigration::BatchedBackgroundMigrationDictionary::Entry,
::Gitlab::Utils::BatchedBackgroundMigrationsDictionary,
finalized_by: '20240104155616'
)
end