root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2024-03-25 09:11:01 +00:00
parent da2b829418
commit ce1d252f5d
17 changed files with 132 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.rubocop_todo/lint/redundant_cop_disable_directive.yml
View File

@ -236,7 +236,6 @@ Lint/RedundantCopDisableDirective:
- 'spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb'
- 'spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb'
- 'spec/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy_spec.rb'
- 'spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb'
- 'spec/lib/gitlab/ci/reports/security/scanner_spec.rb'
- 'spec/lib/gitlab/database/load_balancing/transaction_leaking_spec.rb'
- 'spec/lib/gitlab/database/migration_helpers/restrict_gitlab_schema_spec.rb'

2
.rubocop_todo/rspec/feature_category.yml
View File

@ -2748,7 +2748,6 @@ RSpec/FeatureCategory:
- 'spec/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy_spec.rb'
- 'spec/lib/gitlab/background_migration/batching_strategies/loose_index_scan_batching_strategy_spec.rb'
- 'spec/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy_spec.rb'
- 'spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb'
- 'spec/lib/gitlab/background_migration/copy_column_using_background_migration_job_spec.rb'
- 'spec/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities_spec.rb'
- 'spec/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules_spec.rb'
@ -2764,7 +2763,6 @@ RSpec/FeatureCategory:
- 'spec/lib/gitlab/background_migration/mailers/unconfirm_mailer_spec.rb'
- 'spec/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations_spec.rb'
- 'spec/lib/gitlab/background_migration/populate_projects_star_count_spec.rb'
- 'spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb'
- 'spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb'
- 'spec/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item_spec.rb'
- 'spec/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports_spec.rb'

1
.rubocop_todo/rspec/named_subject.yml
View File

@ -1815,7 +1815,6 @@ RSpec/NamedSubject:
- 'spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb'
- 'spec/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields_spec.rb'
- 'spec/lib/gitlab/background_migration/redis/backfill_project_pipeline_status_ttl_spec.rb'
- 'spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb'
- 'spec/lib/gitlab/background_migration/truncate_overlong_vulnerability_html_titles_spec.rb'
- 'spec/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status_spec.rb'
- 'spec/lib/gitlab/background_task_spec.rb'

1
.rubocop_todo/style/inline_disable_annotation.yml
View File

@ -2805,7 +2805,6 @@ Style/InlineDisableAnnotation:
- 'spec/lib/gitlab/background_migration/batched_migration_job_spec.rb'
- 'spec/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy_spec.rb'
- 'spec/lib/gitlab/background_migration/batching_strategies/loose_index_scan_batching_strategy_spec.rb'
- 'spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb'
- 'spec/lib/gitlab/background_migration/convert_credit_card_validation_data_to_hashes_spec.rb'
- 'spec/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules2_spec.rb'
- 'spec/lib/gitlab/background_migration/delete_orphans_approval_project_rules2_spec.rb'

2
db/docs/ci_job_artifact_states.yml
View File

@ -14,6 +14,6 @@ desired_sharding_key:
backfill_via:
parent:
foreign_key: job_artifact_id
table: ci_job_artifacts
table: p_ci_job_artifacts
sharding_key: project_id
belongs_to: job_artifact

5
doc/ci/ssh_keys/index.md
View File

@ -198,9 +198,10 @@ before_script:
##
## Alternatively, use ssh-keyscan to scan the keys of your private server.
## Replace example.com with your private server's domain name. Repeat that
## command if you have more than one server to connect to.
## command if you have more than one server to connect to. Include the -t
## flag to specify the key type.
##
# - ssh-keyscan example.com >> ~/.ssh/known_hosts
# - ssh-keyscan -t rsa,ed25519 example.com >> ~/.ssh/known_hosts
# - chmod 644 ~/.ssh/known_hosts
##

27
gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/macro_reflection.rb
View File

@ -5,13 +5,38 @@ module ActiveRecord
module Partitioning
module Reflection
module MacroReflection
NO_OWNER = Struct.new(:partition_id).new(1..100_000)
# We override the method to allow eager loading of partitioned records
#
# For eager loading the owner is always nil and we supply a benign
# object that allows the scope to be evaluated with a query like
# `where partition_id between 1 and 100000`
# and transforms it into
# `where partition_id is not null`
# to ensure that no partition is left out by the query.
# This is safe because `partition_id` columns are defined as `not null`
#
def scope_for(relation, owner = nil)
if scope.arity == 1 && owner.nil? && options.key?(:partition_foreign_key)
relation
relation = relation.instance_exec(NO_OWNER, &scope)
if relation_includes_partition_id_condition?(relation)
relation.rewhere(relation.table[:partition_id].not_eq(nil))
else
relation
end
else
super
end
end
def relation_includes_partition_id_condition?(relation)
relation
.where_clause
.extract_attributes
.map(&:name)
.include?('partition_id')
end
end
end
end

9
gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/joins_spec.rb
View File

@ -9,7 +9,8 @@ RSpec.describe 'ActiveRecord::GitlabPatches::Partitioning::Associations::Joins',
join_statement = <<~SQL.squish
SELECT \"pipelines\".*
FROM \"pipelines\"
INNER JOIN \"jobs\" ON \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\"
INNER JOIN \"jobs\" ON \"jobs\".\"partition_id\" IS NOT NULL
AND \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\"
AND \"jobs\".\"partition_id\" = \"pipelines\".\"partition_id\"
WHERE \"pipelines\".\"partition_id\" = #{pipeline.partition_id}
SQL
@ -25,9 +26,11 @@ RSpec.describe 'ActiveRecord::GitlabPatches::Partitioning::Associations::Joins',
join_statement = <<~SQL.squish
SELECT \"pipelines\".*
FROM \"pipelines\"
INNER JOIN \"jobs\" ON \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\"
INNER JOIN \"jobs\" ON \"jobs\".\"partition_id\" IS NOT NULL
AND \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\"
AND \"jobs\".\"partition_id\" = \"pipelines\".\"partition_id\"
INNER JOIN \"metadata\" ON \"metadata\".\"job_id\" = \"jobs\".\"id\"
INNER JOIN \"metadata\" ON \"metadata\".\"partition_id\" IS NOT NULL
AND \"metadata\".\"job_id\" = \"jobs\".\"id\"
AND \"metadata\".\"partition_id\" = \"jobs\".\"partition_id\"
WHERE \"pipelines\".\"partition_id\" = #{pipeline.partition_id}
SQL

76
gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/preloads_spec.rb
View File

@ -216,9 +216,11 @@ RSpec.describe 'ActiveRecord::GitlabPatches::Partitioning::Associations::Preload
AS t2_r2, \"metadata\".\"test_flag\"
AS t2_r3
FROM \"pipelines\"
LEFT OUTER JOIN \"jobs\" ON \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\"
LEFT OUTER JOIN \"jobs\" ON \"jobs\".\"partition_id\" IS NOT NULL
AND \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\"
AND \"jobs\".\"partition_id\" = \"pipelines\".\"partition_id\"
LEFT OUTER JOIN \"metadata\" ON \"metadata\".\"job_id\" = \"jobs\".\"id\"
LEFT OUTER JOIN \"metadata\" ON \"metadata\".\"partition_id\" IS NOT NULL
AND \"metadata\".\"job_id\" = \"jobs\".\"id\"
AND \"metadata\".\"partition_id\" = \"jobs\".\"partition_id\"
WHERE \"pipelines\".\"project_id\" = 1
AND \"pipelines\".\"id\"
@ -237,5 +239,75 @@ RSpec.describe 'ActiveRecord::GitlabPatches::Partitioning::Associations::Preload
expect(result).to include(preload_statement)
end
it 'keeps join conditions from scope' do
preload_statement = <<~SQL.squish
SELECT \"pipelines\".\"id\"
AS t0_r0, \"pipelines\".\"project_id\"
AS t0_r1, \"pipelines\".\"partition_id\"
AS t0_r2, \"jobs\".\"id\"
AS t1_r0, \"jobs\".\"pipeline_id\"
AS t1_r1, \"jobs\".\"partition_id\"
AS t1_r2, \"jobs\".\"name\"
AS t1_r3, \"metadata\".\"id\"
AS t2_r0, \"metadata\".\"job_id\"
AS t2_r1, \"metadata\".\"partition_id\"
AS t2_r2, \"metadata\".\"test_flag\"
AS t2_r3
FROM \"pipelines\"
LEFT OUTER JOIN \"jobs\" ON \"jobs\".\"partition_id\" IS NOT NULL
AND \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\"
AND \"jobs\".\"partition_id\" = \"pipelines\".\"partition_id\"
LEFT OUTER JOIN \"metadata\" ON \"metadata\".\"test_flag\" = 1
AND \"metadata\".\"partition_id\" IS NOT NULL
AND \"metadata\".\"job_id\" = \"jobs\".\"id\"
AND \"metadata\".\"partition_id\" = \"jobs\".\"partition_id\"
WHERE \"pipelines\".\"project_id\" = 1
AND \"pipelines\".\"id\"
IN (#{pipeline.id}, #{other_pipeline.id})
AND \"pipelines\".\"partition_id\" = 100
SQL
result = QueryRecorder.log do
project
.pipelines
.includes(jobs: :test_metadata)
.references(:jobs, :test_metadata)
.where(id: [1, 2], partition_id: 100)
.to_a
end
expect(result).to include(preload_statement)
end
it 'does rewhere the partition_id condition when missing' do
preload_statement = <<~SQL.squish
SELECT \"pipelines\".\"id\"
AS t0_r0, \"pipelines\".\"project_id\"
AS t0_r1, \"pipelines\".\"partition_id\"
AS t0_r2, \"jobs\".\"id\"
AS t1_r0, \"jobs\".\"pipeline_id\"
AS t1_r1, \"jobs\".\"partition_id\"
AS t1_r2, \"jobs\".\"name\"
AS t1_r3 FROM \"pipelines\"
LEFT OUTER JOIN \"jobs\" ON \"jobs\".\"pipeline_id\" = NULL
AND \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\"
AND \"jobs\".\"partition_id\" = \"pipelines\".\"partition_id\"
WHERE \"pipelines\".\"project_id\" = 1
AND \"pipelines\".\"id\" IN (1, 2)
AND \"pipelines\".\"partition_id\" = 100
SQL
result = QueryRecorder.log do
project
.pipelines
.includes(:unpartitioned_jobs)
.references(:unpartitioned_jobs)
.where(id: [1, 2], partition_id: 100)
.to_a
end
expect(result).to include(preload_statement)
end
end
end

13
gems/activerecord-gitlab/spec/support/models.rb
View File

@ -22,6 +22,12 @@ class Pipeline < PartitionedRecord
->(pipeline) { where(partition_id: pipeline.partition_id) },
partition_foreign_key: :partition_id,
dependent: :destroy
has_many :unpartitioned_jobs,
->(pipeline) { where(pipeline: pipeline).order(id: :desc) },
partition_foreign_key: :partition_id,
dependent: :destroy,
class_name: 'Job'
end
class Job < PartitionedRecord
@ -38,6 +44,13 @@ class Job < PartitionedRecord
inverse_of: :job,
autosave: true
has_one :test_metadata,
->(build) { where(partition_id: build.partition_id, test_flag: true) },
foreign_key: :job_id,
partition_foreign_key: :partition_id,
inverse_of: :job,
class_name: 'Metadata'
accepts_nested_attributes_for :metadata
end

12
lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb
View File

@ -1,12 +0,0 @@
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
module BatchingStrategies
# Used to apply additional filters to the batching table, migrated to
# use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96478
class RemoveBackfilledJobArtifactsExpireAtBatchingStrategy < PrimaryKeyBatchingStrategy
end
end
end
end

44
lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb
View File

@ -1,44 +0,0 @@
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This detects and fixes job artifacts that have `expire_at` wrongly backfilled by the migration
# https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723.
# These job artifacts will not be deleted and will have their `expire_at` removed.
class RemoveBackfilledJobArtifactsExpireAt < BatchedMigrationJob
operation_name :update_all
feature_category :database
# The migration would have backfilled `expire_at`
# to midnight on the 22nd of the month of the local timezone,
# storing it as UTC time in the database.
#
# If the timezone setting has changed since the migration,
# the `expire_at` stored in the database could have changed to a different local time other than midnight.
# For example:
# - changing timezone from UTC+02:00 to UTC+02:30 would change the `expire_at` in local time 00:00:00 to 00:30:00.
# - changing timezone from UTC+00:00 to UTC-01:00 would change the `expire_at` in local time 00:00:00 to 23:00:00
# on the previous day (21st).
#
# Therefore job artifacts that have `expire_at` exactly on the 00, 30 or 45 minute mark
# on the dates 21, 22, 23 of the month will not be deleted.
# https://en.wikipedia.org/wiki/List_of_UTC_time_offsets
EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE = <<~SQL
EXTRACT(day FROM timezone('UTC', expire_at)) IN (21, 22, 23)
AND EXTRACT(minute FROM timezone('UTC', expire_at)) IN (0, 30, 45)
AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0
SQL
scope_to ->(relation) {
relation.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE)
.or(relation.where(file_type: 3))
}
def perform
each_sub_batch do |sub_batch|
sub_batch.update_all(expire_at: nil)
end
end
end
end
end

8
lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb
View File

@ -8,7 +8,13 @@ module Gitlab
class PartitioningRoutingAnalyzer < Database::QueryAnalyzers::Base
RoutingTableNotUsedError = Class.new(QueryAnalyzerError)
ENABLED_TABLES = %w[ci_builds ci_builds_metadata].freeze
ENABLED_TABLES = %w[
ci_builds
ci_builds_metadata
ci_job_artifacts
ci_pipeline_variables
ci_stages
].freeze
class << self
def enabled?

7
spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb
View File

@ -1,7 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::RemoveBackfilledJobArtifactsExpireAtBatchingStrategy do # rubocop:disable Layout/LineLength
it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchingStrategy }
end

96
spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb
View File

@ -1,96 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::BackgroundMigration::RemoveBackfilledJobArtifactsExpireAt do
it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchedMigrationJob }
describe '#perform' do
let(:job_artifact) { table(:ci_job_artifacts, database: :ci) }
let(:jobs) { table(:p_ci_builds, database: :ci) { |model| model.primary_key = :id } }
let(:test_worker) do
described_class.new(
start_id: 1,
end_id: 100,
batch_table: :ci_job_artifacts,
batch_column: :id,
sub_batch_size: 10,
pause_ms: 0,
connection: Ci::ApplicationRecord.connection
)
end
let!(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') }
let!(:project) do
table(:projects).create!(
id: 1,
name: 'gitlab1',
path: 'gitlab1',
project_namespace_id: 1,
namespace_id: namespace.id
)
end
subject { test_worker.perform }
context 'with artifacts that has backfilled expire_at' do
let!(:created_on_00_30_45_minutes_on_21_22_23) do
create_job_artifact(id: 1, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.000'))
create_job_artifact(id: 2, file_type: 1, expire_at: Time.zone.parse('2022-01-21 01:30:00.000'))
create_job_artifact(id: 3, file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:00:00.000'))
create_job_artifact(id: 4, file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:30:00.000'))
create_job_artifact(id: 5, file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:00:00.000'))
create_job_artifact(id: 6, file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:30:00.000'))
create_job_artifact(id: 7, file_type: 1, expire_at: Time.zone.parse('2022-01-23 06:45:00.000'))
end
let!(:created_close_to_00_or_30_minutes) do
create_job_artifact(id: 8, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.001'))
create_job_artifact(id: 9, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:30:00.999'))
end
let!(:created_on_00_or_30_minutes_on_other_dates) do
create_job_artifact(id: 10, file_type: 1, expire_at: Time.zone.parse('2022-01-01 00:00:00.000'))
create_job_artifact(id: 11, file_type: 1, expire_at: Time.zone.parse('2022-01-19 12:00:00.000'))
create_job_artifact(id: 12, file_type: 1, expire_at: Time.zone.parse('2022-01-24 23:30:00.000'))
end
let!(:created_at_other_times) do
create_job_artifact(id: 13, file_type: 1, expire_at: Time.zone.parse('2022-01-19 00:00:00.000'))
create_job_artifact(id: 14, file_type: 1, expire_at: Time.zone.parse('2022-01-19 00:30:00.000'))
create_job_artifact(id: 15, file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:00:00.000'))
create_job_artifact(id: 16, file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:30:00.000'))
end
it 'removes expire_at on job artifacts that have expire_at on 00, 30 or 45 minute of 21, 22, 23 of the month' do
expect { subject }.to change { job_artifact.where(expire_at: nil).count }.from(0).to(7)
end
it 'keeps expire_at on other job artifacts' do
expect { subject }.to change { job_artifact.where.not(expire_at: nil).count }.from(16).to(9)
end
end
context 'with trace artifacts that has backfilled expire_at' do
let!(:trace_artifacts) do
create_job_artifact(id: 1, file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000'))
create_job_artifact(id: 2, file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000'))
end
it 'removes expire_at on trace job artifacts' do
expect { subject }.to change { job_artifact.where(expire_at: nil).count }.from(0).to(2)
end
end
private
def create_job_artifact(id:, file_type:, expire_at:)
job = jobs.create!(partition_id: 100)
job_artifact.create!(
id: id, job_id: job.id, expire_at: expire_at, project_id: project.id,
file_type: file_type, partition_id: 100
)
end
end
end

2
spec/migrations/20240108072318_fix_invalid_records_ci_build_trace_metadata_spec.rb
View File

@ -3,7 +3,7 @@
require 'spec_helper'
require_migration!
RSpec.describe FixInvalidRecordsCiBuildTraceMetadata, migration: :gitlab_ci, feature_category: :continuous_integration do
RSpec.describe FixInvalidRecordsCiBuildTraceMetadata, :suppress_partitioning_routing_analyzer, migration: :gitlab_ci, feature_category: :continuous_integration do
let(:trace_metadata) { table(:ci_build_trace_metadata) }
let(:job_artifacts) { table(:ci_job_artifacts) }
let(:connection) { job_artifacts.connection }

2
spec/migrations/20240108072545_fix_invalid_records_ci_job_artifact_states_spec.rb
View File

@ -3,7 +3,7 @@
require 'spec_helper'
require_migration!
RSpec.describe FixInvalidRecordsCiJobArtifactStates, migration: :gitlab_ci, feature_category: :continuous_integration do
RSpec.describe FixInvalidRecordsCiJobArtifactStates, :suppress_partitioning_routing_analyzer, migration: :gitlab_ci, feature_category: :continuous_integration do
let(:artifact_state) { table(:ci_job_artifact_states) }
let(:job_artifacts) { table(:ci_job_artifacts) }
let(:connection) { job_artifacts.connection }