-
### Behavior change for protected variables and multi-project pipelines
diff --git a/lib/gitlab/background_migration/backfill_upstream_pipeline_partition_id_on_p_ci_builds.rb b/lib/gitlab/background_migration/backfill_upstream_pipeline_partition_id_on_p_ci_builds.rb
new file mode 100644
index 00000000000..d40a5ba59e8
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_upstream_pipeline_partition_id_on_p_ci_builds.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ class BackfillUpstreamPipelinePartitionIdOnPCiBuilds < BatchedMigrationJob
+ operation_name :update_all
+ feature_category :continuous_integration
+
+ def perform
+ distinct_each_batch do |batch|
+ pipeline_ids = batch.pluck(batch_column)
+ base_relation
+ .where(batch_column => pipeline_ids)
+ .where('p_ci_builds.upstream_pipeline_id = ci_pipelines.id')
+ .update_all('upstream_pipeline_partition_id = ci_pipelines.partition_id FROM ci_pipelines')
+ end
+ end
+
+ private
+
+ def base_relation
+ define_batchable_model(batch_table, connection: connection, primary_key: :id)
+ .where(batch_column => start_id..end_id)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index d899ed3ba25..49d3eaa576f 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -432,7 +432,7 @@ module Gitlab
@committer_name = commit.committer.name.dup
@committer_email = commit.committer.email.dup
@parent_ids = Array(commit.parent_ids)
- @trailers = commit.trailers.to_h { |t| [t.key, t.value] }
+ @trailers = commit.trailers.to_h { |t| [t.key, encode!(t.value)] }
@extended_trailers = parse_commit_trailers(commit.trailers)
@referenced_by = Array(commit.referenced_by)
end
@@ -440,7 +440,7 @@ module Gitlab
# Turn the commit trailers into a hash of key: [value, value] arrays
def parse_commit_trailers(trailers)
trailers.each_with_object({}) do |trailer, hash|
- (hash[trailer.key] ||= []) << trailer.value
+ (hash[trailer.key] ||= []) << encode!(trailer.value)
end
end
diff --git a/lib/gitlab/wiki_file_finder.rb b/lib/gitlab/wiki_file_finder.rb
index a963cc7954f..60ceb342875 100644
--- a/lib/gitlab/wiki_file_finder.rb
+++ b/lib/gitlab/wiki_file_finder.rb
@@ -2,6 +2,8 @@
module Gitlab
class WikiFileFinder < FileFinder
+ extend ::Gitlab::Utils::Override
+
attr_reader :repository
def initialize(project, ref)
@@ -12,12 +14,24 @@ module Gitlab
private
+ override :search_paths
def search_paths(query)
safe_query = Regexp.escape(query.tr(' ', '-'))
safe_query = Regexp.new(safe_query, Regexp::IGNORECASE)
paths = repository.ls_files(ref)
+ paths.select { |path| valid_format_wiki_file?(path) }.grep(safe_query)
+ end
- paths.grep(safe_query)
+ override :find_by_content
+ def find_by_content(query, options)
+ repository.search_files_by_content(query, ref, options).filter_map do |result|
+ blob = Gitlab::Search::FoundBlob.new(content_match: result, project: project, ref: ref, repository: repository)
+ blob if valid_format_wiki_file?(blob.path)
+ end
+ end
+
+ def valid_format_wiki_file?(path)
+ Wiki::MARKUPS.values.pluck(:extension_regex).any? { |regex| regex.match?(path) } # rubocop: disable CodeReuse/ActiveRecord -- Not an ActiveRecord
end
end
end
diff --git a/qa/qa/specs/features/api/10_govern/project_access_token_spec.rb b/qa/qa/specs/features/api/10_govern/project_access_token_spec.rb
index 42c12950367..e1aff18ffcc 100644
--- a/qa/qa/specs/features/api/10_govern/project_access_token_spec.rb
+++ b/qa/qa/specs/features/api/10_govern/project_access_token_spec.rb
@@ -30,7 +30,7 @@ module QA
end.not_to raise_error
end
- it 'can be used to commit via the API',
+ it 'can be used to commit via the API', :blocking,
testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347859' do
expect do
create(:commit,
diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb
index d3677c008a8..fce630ab1a1 100644
--- a/spec/db/schema_spec.rb
+++ b/spec/db/schema_spec.rb
@@ -67,7 +67,7 @@ RSpec.describe 'Database schema', feature_category: :database do
broadcast_messages: %w[namespace_id],
chat_names: %w[chat_id team_id user_id],
chat_teams: %w[team_id],
- ci_builds: %w[project_id runner_id user_id erased_by_id trigger_request_id partition_id auto_canceled_by_partition_id execution_config_id],
+ ci_builds: %w[project_id runner_id user_id erased_by_id trigger_request_id partition_id auto_canceled_by_partition_id execution_config_id upstream_pipeline_partition_id],
ci_daily_build_group_report_results: %w[partition_id],
ci_job_artifacts: %w[partition_id project_id job_id],
ci_namespace_monthly_usages: %w[namespace_id],
@@ -129,7 +129,7 @@ RSpec.describe 'Database schema', feature_category: :database do
oauth_access_tokens: %w[resource_owner_id application_id],
oauth_applications: %w[owner_id],
oauth_device_grants: %w[resource_owner_id application_id],
- p_ci_builds: %w[erased_by_id trigger_request_id partition_id auto_canceled_by_partition_id execution_config_id],
+ p_ci_builds: %w[erased_by_id trigger_request_id partition_id auto_canceled_by_partition_id execution_config_id upstream_pipeline_partition_id],
p_batched_git_ref_updates_deletions: %w[project_id partition_id],
p_catalog_resource_sync_events: %w[catalog_resource_id project_id partition_id],
p_catalog_resource_component_usages: %w[used_by_project_id], # No FK constraint because we want to preserve historical usage data
diff --git a/spec/lib/gitlab/background_migration/backfill_upstream_pipeline_partition_id_on_p_ci_builds_spec.rb b/spec/lib/gitlab/background_migration/backfill_upstream_pipeline_partition_id_on_p_ci_builds_spec.rb
new file mode 100644
index 00000000000..2e59edec2d8
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/backfill_upstream_pipeline_partition_id_on_p_ci_builds_spec.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillUpstreamPipelinePartitionIdOnPCiBuilds, feature_category: :continuous_integration do
+ let(:pipelines_table) { table(:ci_pipelines, database: :ci) { |t| t.primary_key = :id } }
+ let(:jobs_table) { table(:p_ci_builds, database: :ci) { |t| t.primary_key = :id } }
+
+ let!(:pipeline_1) { pipelines_table.create!(partition_id: 100, project_id: 1) }
+ let!(:pipeline_2) { pipelines_table.create!(partition_id: 100, project_id: 1) }
+ let!(:pipeline_3) { pipelines_table.create!(partition_id: 100, project_id: 1) }
+
+ let!(:job_1) { jobs_table.create!(commit_id: pipeline_1.id, partition_id: pipeline_1.partition_id, project_id: 1) }
+ let!(:job_2) { jobs_table.create!(commit_id: pipeline_2.id, partition_id: pipeline_2.partition_id, project_id: 1) }
+ let!(:job_3) { jobs_table.create!(commit_id: pipeline_2.id, partition_id: pipeline_2.partition_id, project_id: 1) }
+
+ let(:migration_attrs) do
+ {
+ start_id: jobs_table.minimum(:upstream_pipeline_id),
+ end_id: jobs_table.maximum(:upstream_pipeline_id),
+ batch_table: :p_ci_builds,
+ batch_column: :upstream_pipeline_id,
+ sub_batch_size: 2,
+ pause_ms: 0,
+ connection: connection
+ }
+ end
+
+ let!(:migration) { described_class.new(**migration_attrs) }
+ let(:connection) { Ci::ApplicationRecord.connection }
+
+ around do |example|
+ connection.transaction do
+ connection.execute(<<~SQL)
+ ALTER TABLE ci_pipelines DISABLE TRIGGER ALL;
+ SQL
+
+ example.run
+
+ connection.execute(<<~SQL)
+ ALTER TABLE ci_pipelines ENABLE TRIGGER ALL;
+ SQL
+ end
+ end
+
+ describe '#perform' do
+ before do
+ job_2.update!(upstream_pipeline_id: pipeline_1.id)
+ job_3.update!(upstream_pipeline_id: pipeline_3.id)
+ pipeline_3.update!(partition_id: 101)
+ end
+
+ it 'backfills upstream_pipeline_partition_id' do
+ expect { migration.perform }
+ .to not_change { job_1.reload.upstream_pipeline_partition_id }
+ .and change { job_2.reload.upstream_pipeline_partition_id }.from(nil).to(100)
+ .and change { job_3.reload.upstream_pipeline_partition_id }.from(nil).to(101)
+ end
+ end
+end
diff --git a/spec/lib/gitlab/git/commit_spec.rb b/spec/lib/gitlab/git/commit_spec.rb
index a924137b8ec..767ca360b96 100644
--- a/spec/lib/gitlab/git/commit_spec.rb
+++ b/spec/lib/gitlab/git/commit_spec.rb
@@ -51,6 +51,30 @@ RSpec.describe Gitlab::Git::Commit, feature_category: :source_code_management do
)
end
+ context 'non-ASCII content' do
+ let(:body) do
+ body = +<<~BODY
+ Äpfel
+
+ Changelog: Äpfel
+ BODY
+
+ [subject, "\n", body.force_encoding("ASCII-8BIT")].join
+ end
+
+ it "parses non-ASCII commit trailers" do
+ expect(commit.trailers).to eq(
+ { 'Changelog' => 'Äpfel' }
+ )
+ end
+
+ it "parses non-ASCII extended commit trailers" do
+ expect(commit.extended_trailers).to eq(
+ { 'Changelog' => ['Äpfel'] }
+ )
+ end
+ end
+
context 'non-UTC dates' do
let(:seconds) { Time.now.to_i }
diff --git a/spec/lib/gitlab/wiki_file_finder_spec.rb b/spec/lib/gitlab/wiki_file_finder_spec.rb
index 3102f628de9..644d718c2a6 100644
--- a/spec/lib/gitlab/wiki_file_finder_spec.rb
+++ b/spec/lib/gitlab/wiki_file_finder_spec.rb
@@ -2,7 +2,7 @@
require 'spec_helper'
-RSpec.describe Gitlab::WikiFileFinder do
+RSpec.describe Gitlab::WikiFileFinder, feature_category: :global_search do
describe '#find' do
let_it_be(:project) do
create(:project, :public, :wiki_repo).tap do |project|
@@ -17,5 +17,10 @@ RSpec.describe Gitlab::WikiFileFinder do
let(:expected_file_by_path) { 'Files/Title.md' }
let(:expected_file_by_content) { 'CHANGELOG.md' }
end
+
+ it 'does not include .gitlab/redirects.yml' do
+ results = described_class.new(project, project.wiki.default_branch).find(Wiki::REDIRECTS_YML)
+ expect(results.map(&:blob_path)).not_to include Wiki::REDIRECTS_YML
+ end
end
end
diff --git a/spec/migrations/20240708105034_queue_backfill_upstream_pipeline_partition_id_on_p_ci_builds_spec.rb b/spec/migrations/20240708105034_queue_backfill_upstream_pipeline_partition_id_on_p_ci_builds_spec.rb
new file mode 100644
index 00000000000..06828032c3c
--- /dev/null
+++ b/spec/migrations/20240708105034_queue_backfill_upstream_pipeline_partition_id_on_p_ci_builds_spec.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueBackfillUpstreamPipelinePartitionIdOnPCiBuilds, migration: :gitlab_ci, feature_category: :continuous_integration do
+ let!(:batched_migration) { described_class::MIGRATION }
+
+ it 'schedules a new batched migration' do
+ reversible_migration do |migration|
+ migration.before -> {
+ expect(batched_migration).not_to have_scheduled_batched_migration
+ }
+
+ migration.after -> {
+ expect(batched_migration).to have_scheduled_batched_migration(
+ table_name: :p_ci_builds,
+ column_name: :upstream_pipeline_id,
+ gitlab_schema: :gitlab_ci,
+ interval: described_class::DELAY_INTERVAL,
+ batch_size: described_class::BATCH_SIZE,
+ sub_batch_size: described_class::SUB_BATCH_SIZE
+ )
+ }
+ end
+ end
+end
diff --git a/spec/models/ci/processable_spec.rb b/spec/models/ci/processable_spec.rb
index e75e80d98a6..225e08b65c6 100644
--- a/spec/models/ci/processable_spec.rb
+++ b/spec/models/ci/processable_spec.rb
@@ -92,7 +92,8 @@ RSpec.describe Ci::Processable, feature_category: :continuous_integration do
runner_id tag_taggings taggings tags trigger_request_id
user_id auto_canceled_by_id retried failure_reason
sourced_pipelines sourced_pipeline artifacts_file_store artifacts_metadata_store
- metadata runner_manager_build runner_manager runner_session trace_chunks upstream_pipeline_id
+ metadata runner_manager_build runner_manager runner_session trace_chunks
+ upstream_pipeline_id upstream_pipeline_partition_id
artifacts_file artifacts_metadata artifacts_size commands
resource resource_group_id processed security_scans author
pipeline_id report_results pending_state pages_deployments