Add latest changes from gitlab-org/gitlab@master

app/serializers/codequality_degradation_entity.rb

@@ -10,7 +10,7 @@ class CodequalityDegradationEntity < Grape::Entity
   end
 
   expose :file_path do |degradation|
-    degradation.dig(:location, :path)
+    degradation.dig(:location, :path).sub(%r{^./}, '')
   end
 
   expose :line do |degradation|

db/fixtures/development/18_abuse_reports.rb

@@ -5,7 +5,7 @@ module Db
         def self.seed
           Gitlab::Seeder.quiet do
             (::AbuseReport.default_per_page + 3).times do |i|
-              username = "#{::Gitlab::Seeder::REPORTED_USER_START}#{FFaker::Internet.unique.user_name}"
+              username = "#{::Gitlab::Seeder::REPORTED_USER_START}#{::Gitlab::Faker::Internet.unique_username}"
               reported_user =
                 ::User.create!(
                   username: username,

db/migrate/20250410083733_add_pre_approved_agent_privileges_to_duo_workflows_workflows.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class AddPreApprovedAgentPrivilegesToDuoWorkflowsWorkflows < Gitlab::Database::Migration[2.2]
+  milestone '18.0'
+
+  READ_WRITE_FILES = 1
+  READ_ONLY_GITLAB = 2
+
+  def change
+    add_column :duo_workflows_workflows, :pre_approved_agent_privileges, :smallint, array: true,
+      default: [READ_WRITE_FILES, READ_ONLY_GITLAB], null: false
+  end
+end

db/schema_migrations/20250410083733

@@ -0,0 +1 @@
+6f1737dfd78aa35e399cd34786ea29cab06789560879ff8d23463bd840aeab3f

db/structure.sql

@@ -14184,6 +14184,7 @@ CREATE TABLE duo_workflows_workflows (
     agent_privileges smallint[] DEFAULT '{1,2}'::smallint[] NOT NULL,
     workflow_definition text DEFAULT 'software_development'::text NOT NULL,
     allow_agent_to_request_user boolean DEFAULT true NOT NULL,
+    pre_approved_agent_privileges smallint[] DEFAULT '{1,2}'::smallint[] NOT NULL,
     CONSTRAINT check_5aedde451d CHECK ((char_length(goal) <= 4096)),
     CONSTRAINT check_ec723e2a1a CHECK ((char_length(workflow_definition) <= 255))
 );

doc/ci/yaml/_index.md

@@ -1869,7 +1869,7 @@ cache-job:
 
 ##### `cache:key:files`
 
-Use the `cache:key:files` keyword to generate a new key when one or two specific files
+Use the `cache:key:files` keyword to generate a new key when files matching either of the defined paths or patterns
 change. `cache:key:files` lets you reuse some caches, and rebuild them less often,
 which speeds up subsequent pipeline runs.
 
@@ -1878,7 +1878,7 @@ which speeds up subsequent pipeline runs.
 
 **Supported values**:
 
-- An array of one or two file paths.
+- An array of up to two file paths or patterns.
 
 CI/CD variables are not supported.
 
@@ -1909,6 +1909,8 @@ use the new cache, instead of rebuilding the dependencies.
 - The cache `key` is a SHA computed from the most recent commits
   that changed each listed file.
   If neither file is changed in any commits, the fallback key is `default`.
+- Wildcard patterns like `**/package.json` can be used. An [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/301161)
+  exists to increase the number of paths or patterns allowed for a cache key.
 
 ##### `cache:key:prefix`
 

lib/gitlab/database/tables_locker.rb

@@ -5,15 +5,20 @@ module Gitlab
     class TablesLocker
       GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_embedding gitlab_geo gitlab_jh].freeze
 
-      def initialize(logger: nil, dry_run: false, include_partitions: true)
+      def initialize(logger: nil, dry_run: false, include_partitions: true, options: {})
         @logger = logger
         @dry_run = dry_run
         @result = []
         @include_partitions = include_partitions
+        @scope_to_database = options[:scope_to_database].to_s
+
+        validate_scopes
       end
 
       def unlock_writes
         Gitlab::Database::EachDatabase.each_connection do |connection, database_name|
+          next if skip_connection?(database_name)
+
           tables_to_lock(connection) do |table_name, schema_name|
             # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834
             next if schema_name.in? GITLAB_SCHEMAS_TO_IGNORE
@@ -29,6 +34,8 @@ module Gitlab
       # on the database where they belong
       def lock_writes
         Gitlab::Database::EachDatabase.each_connection(include_shared: false) do |connection, database_name|
+          next if skip_connection?(database_name)
+
           schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection)
 
           tables_to_lock(connection) do |table_name, schema_name|
@@ -48,6 +55,19 @@ module Gitlab
 
       private
 
+      def skip_connection?(database_name)
+        return false if @scope_to_database.empty? || database_name == @scope_to_database
+
+        true
+      end
+
+      def validate_scopes
+        return if @scope_to_database.blank? ||
+          Gitlab::Database.all_database_connections.key?(@scope_to_database)
+
+        raise "#{@scope_to_database} is not a valid database to scope to."
+      end
+
       # Unlocks the writes on the table and its partitions
       def unlock_writes_on_table(table_name, connection, database_name)
         @result << lock_writes_manager(table_name, connection, database_name).unlock_writes

lib/tasks/gitlab/db/lock_writes.rake

@@ -6,10 +6,15 @@ namespace :gitlab do
     task lock_writes: [:environment, 'gitlab:db:validate_config'] do
       logger = Logger.new($stdout)
       logger.level = Gitlab::Utils.to_boolean(ENV['VERBOSE']) ? Logger::INFO : Logger::WARN
+
+      options = {}
+      options[:scope_to_database] = ENV['SCOPE_TO_DATABASE'] if ENV['SCOPE_TO_DATABASE']
+
       Gitlab::Database::TablesLocker.new(
         logger: logger,
         dry_run: Gitlab::Utils.to_boolean(ENV['DRY_RUN'], default: false),
-        include_partitions: Gitlab::Utils.to_boolean(ENV['INCLUDE_PARTITIONS'], default: true)
+        include_partitions: Gitlab::Utils.to_boolean(ENV['INCLUDE_PARTITIONS'], default: true),
+        options: options
       ).lock_writes
     end
 
@@ -17,10 +22,15 @@ namespace :gitlab do
     task unlock_writes: :environment do
       logger = Logger.new($stdout)
       logger.level = Gitlab::Utils.to_boolean(ENV['VERBOSE']) ? Logger::INFO : Logger::WARN
+
+      options = {}
+      options[:scope_to_database] = ENV['SCOPE_TO_DATABASE'] if ENV['SCOPE_TO_DATABASE']
+
       Gitlab::Database::TablesLocker.new(
         logger: logger,
         dry_run: Gitlab::Utils.to_boolean(ENV['DRY_RUN'], default: false),
-        include_partitions: Gitlab::Utils.to_boolean(ENV['INCLUDE_PARTITIONS'], default: true)
+        include_partitions: Gitlab::Utils.to_boolean(ENV['INCLUDE_PARTITIONS'], default: true),
+        options: options
       ).unlock_writes
     end
   end

scripts/rspec_helpers.sh

@@ -272,7 +272,7 @@ function rspec_parallelized_job() {
   local test_level="${job_name[1]}"
   # e.g. 'rspec unit pg14 1/24 278964' would become 'rspec_unit_pg14_1_24_278964'
   local report_name=$(echo "${CI_JOB_NAME} ${CI_PROJECT_ID}" | sed -E 's|[/ ]|_|g')
-  local rspec_opts="${1:-}"
+  local rspec_opts="--force-color ${1:-}"
   local rspec_tests_mapping_enabled="${RSPEC_TESTS_MAPPING_ENABLED:-}"
   local spec_folder_prefixes=""
   local rspec_flaky_folder_path="$(dirname "${FLAKY_RSPEC_SUITE_REPORT_PATH}")/"

spec/lib/gitlab/database/tables_locker_spec.rb

@@ -240,6 +240,32 @@ RSpec.describe Gitlab::Database::TablesLocker, :suppress_gitlab_schemas_validate
       gitlab_main_detached_partition = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_gitlab_main_part_20220101"
       it_behaves_like 'unlock partitions', gitlab_main_detached_partition, 'main'
       it_behaves_like 'lock partitions', gitlab_main_detached_partition, 'ci'
+
+      context 'when scope_to_database is set' do
+        subject { described_class.new(options: { scope_to_database: :main }).lock_writes }
+
+        it_behaves_like 'lock tables', :gitlab_ci, 'main'
+
+        it_behaves_like 'unlock tables', :gitlab_main_clusterwide, 'main'
+        it_behaves_like 'unlock tables', :gitlab_main, 'main'
+        it_behaves_like 'unlock tables', :gitlab_shared, 'main'
+        it_behaves_like 'unlock tables', :gitlab_internal, 'main'
+
+        gitlab_main_partition = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.zoekt_tasks_test_partition"
+        it_behaves_like 'unlock partitions', gitlab_main_partition, 'main'
+
+        gitlab_main_detached_partition =
+          "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_gitlab_main_part_20220101"
+        it_behaves_like 'unlock partitions', gitlab_main_detached_partition, 'main'
+      end
+
+      context 'when an invalid database scope has been passed' do
+        subject { described_class.new(options: { scope_to_database: :unknown }).lock_writes }
+
+        it 'raises an error' do
+          expect { subject }.to raise_error(RuntimeError, 'unknown is not a valid database to scope to.')
+        end
+      end
     end
 
     describe '#unlock_writes' do
@@ -261,6 +287,22 @@ RSpec.describe Gitlab::Database::TablesLocker, :suppress_gitlab_schemas_validate
       gitlab_main_detached_partition = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_gitlab_main_part_20220101"
       it_behaves_like 'unlock partitions', gitlab_main_detached_partition, 'main'
       it_behaves_like 'unlock partitions', gitlab_main_detached_partition, 'ci'
+
+      context 'when scope_to_database is set' do
+        subject { described_class.new(options: { scope_to_database: :main }).unlock_writes }
+
+        it_behaves_like "unlock tables", :gitlab_ci, 'main'
+        it_behaves_like "unlock tables", :gitlab_main, 'main'
+        it_behaves_like "unlock tables", :gitlab_shared, 'main'
+        it_behaves_like "unlock tables", :gitlab_internal, 'main'
+
+        gitlab_main_partition = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.zoekt_tasks_test_partition"
+        it_behaves_like 'unlock partitions', gitlab_main_partition, 'main'
+
+        gitlab_main_detached_partition =
+          "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}._test_gitlab_main_part_20220101"
+        it_behaves_like 'unlock partitions', gitlab_main_detached_partition, 'main'
+      end
     end
 
     context 'when not including partitions' do

spec/serializers/codequality_degradation_entity_spec.rb

@@ -65,6 +65,18 @@ RSpec.describe CodequalityDegradationEntity, feature_category: :code_quality do
           expect(subject[:severity]).to eq('unknown')
         end
       end
+
+      context 'when file path starts with ./' do
+        let(:codequality_degradation) { build(:codequality_degradation_3) }
+
+        before do
+          codequality_degradation[:location][:path] = './file_b.rb'
+        end
+
+        it 'removes the ./', :aggregate_failures do
+          expect(subject[:file_path]).to eq("file_b.rb")
+        end
+      end
     end
   end
 end

spec/support/rspec_run_time.rb

@@ -2,6 +2,7 @@
 
 require 'os'
 require 'yaml'
+require 'etc'
 require 'rspec/core/formatters/base_formatter'
 require_relative '../../tooling/lib/tooling/helpers/duration_formatter'
 
@@ -46,8 +47,8 @@ module Support
           actual_duration = time_now - @current_group_start_time
 
           output.puts "\n# [RSpecRunTime] Finishing example group #{file_path}. " \
-                      "It took #{readable_duration(actual_duration)}. " \
-                      "#{expected_run_time(file_path)}"
+            "It took #{readable_duration(actual_duration)}. " \
+            "#{expected_run_time(file_path)}"
         end
 
         output_elapsed_time
@@ -55,6 +56,10 @@ module Support
 
       private
 
+      def nprocessors
+        @nprocessors ||= Etc.nprocessors
+      end
+
       def expected_duration_report
         report_path = ENV['KNAPSACK_RSPEC_SUITE_REPORT_PATH']
 
@@ -87,9 +92,9 @@ module Support
         unless @last_elapsed_seconds.nil? || elapsed_seconds - @last_elapsed_seconds < 1
           output.puts \
             "# [RSpecRunTime] RSpec elapsed time: #{readable_duration(elapsed_seconds)}. " \
-            "#{current_rss_in_megabytes}. " \
-            "Threads: #{threads_count}. " \
-            "#{load_average}.\n\n" \
+              "#{current_rss_in_megabytes}. " \
+              "Threads: #{threads_count}. " \
+              "#{load_average}.\n\n" \
         end
 
         @last_elapsed_seconds = elapsed_seconds
@@ -102,11 +107,13 @@ module Support
       end
 
       def load_average
-        if File.exist?('/proc/loadavg')
-          "load average: #{File.read('/proc/loadavg')}"
-        else
-          `uptime`[/(load average:[^\n]+)/, 1] || '(uptime failed)'
-        end
+        load_avg = if File.exist?('/proc/loadavg')
+                     "load average: #{File.read('/proc/loadavg')}"
+                   else
+                     `uptime`[/(load average:[^\n]+)/, 1] || '(uptime failed)'
+                   end
+
+        "#{load_avg}, available cpu cores: #{nprocessors}"
       end
 
       def threads_count

spec/tasks/gitlab/db/lock_writes_rake_spec.rb

@@ -15,11 +15,15 @@ RSpec.describe 'gitlab:db:lock_writes', :reestablished_active_record_base, featu
   let(:dry_run) { false }
   let(:verbose) { false }
   let(:include_partitions) { true }
+  let(:options) { {} }
 
   before do
     allow(Logger).to receive(:new).with($stdout).and_return(logger)
     allow(Gitlab::Database::TablesLocker).to receive(:new).with(
-      logger: logger, dry_run: dry_run, include_partitions: include_partitions
+      logger: logger,
+      dry_run: dry_run,
+      include_partitions: include_partitions,
+      options: options
     ).and_return(table_locker)
   end
 
@@ -45,6 +49,16 @@ RSpec.describe 'gitlab:db:lock_writes', :reestablished_active_record_base, featu
       include_examples "call table locker", :lock_writes
     end
 
+    context 'when environment sets SCOPE_TO_DATABASE' do
+      let(:options) { { scope_to_database: 'main' } }
+
+      before do
+        stub_env('SCOPE_TO_DATABASE', 'main')
+      end
+
+      include_examples "call table locker", :lock_writes
+    end
+
     context 'when environment sets INCLUDE_PARTITIONS to false' do
       let(:include_partitions) { false }
 
@@ -107,6 +121,16 @@ RSpec.describe 'gitlab:db:lock_writes', :reestablished_active_record_base, featu
       include_examples "call table locker", :unlock_writes
     end
 
+    context 'when environment sets SCOPE_TO_DATABASE' do
+      let(:options) { { scope_to_database: 'main' } }
+
+      before do
+        stub_env('SCOPE_TO_DATABASE', 'main')
+      end
+
+      include_examples "call table locker", :unlock_writes
+    end
+
     context 'when environment sets INCLUDE_PARTITIONS to false' do
       let(:include_partitions) { false }