Add latest changes from gitlab-org/gitlab@master

GITLAB_KAS_VERSION

@@ -1 +1 @@
-a92cf38ec9c34a3ccb4d8c07acadc3935fd250ef
+9db831dfcab1ec12908d895f396f1bcb18a219e7

app/models/packages/dependency.rb

@@ -6,15 +6,11 @@ class Packages::Dependency < ApplicationRecord
   has_many :dependency_links, class_name: 'Packages::DependencyLink'
   belongs_to :project
 
-  validates :name, :version_pattern, presence: true
+  validates :name, :version_pattern, :project_id, presence: true
 
   validates :name, uniqueness: { scope: :version_pattern }, unless: :project_id
   validates :name, uniqueness: { scope: %i[version_pattern project_id] }, if: :project_id
 
-  # TODO: remove the update operation when all packages dependencies have a `project_id`.
-  # https://gitlab.com/gitlab-org/gitlab/-/issues/481541
-  scope :without_project, -> { where(project_id: nil) }
-
   NAME_VERSION_PATTERN_TUPLE_MATCHING = '(name, version_pattern) = (?, ?)'
   MAX_STRING_LENGTH = 255
   MAX_CHUNKED_QUERIES_COUNT = 10
@@ -27,13 +23,8 @@ class Packages::Dependency < ApplicationRecord
     names_and_version_patterns.each_slice(chunk_size) do |tuples|
       where_statement = Array.new(tuples.size, NAME_VERSION_PATTERN_TUPLE_MATCHING)
                              .join(' OR ')
-      # Additionally, we look up for dependencies with `project_id IS NULL` to avoid creating duplicates:
-      # a dependency with `project_id` and the same dependency without `project_id`.
-      #
-      # TODO: remove `nil` value when all packages dependencies have a `project_id`.
-      # https://gitlab.com/gitlab-org/gitlab/-/issues/481541
       ids = where(where_statement, *tuples.flatten)
-              .where(project_id: [project_id, nil])
+              .where(project_id: project_id)
               .limit(max_rows_limit + 1)
               .pluck(:id)
       matched_ids.concat(ids)

app/services/packages/create_dependency_service.rb

@@ -34,11 +34,6 @@ module Packages
       ApplicationRecord.transaction do
         inserted_ids = bulk_insert_package_dependencies(dependencies_to_insert)
         bulk_insert_package_dependency_links(type, (existing_ids + inserted_ids))
-        # TODO: remove the update operation when all packages dependencies have a `project_id`.
-        # https://gitlab.com/gitlab-org/gitlab/-/issues/481541
-        if Packages::Dependency.id_in(existing_ids).without_project.any?
-          bulk_update_project_id_package_dependencies(existing_ids)
-        end
       end
     end
 
@@ -84,10 +79,6 @@ module Packages
 
       ApplicationRecord.legacy_bulk_insert(Packages::DependencyLink.table_name, rows)
     end
-
-    def bulk_update_project_id_package_dependencies(ids)
-      Packages::Dependency.id_in(ids).without_project.update_all(project_id: project_id)
-    end
   end
   # rubocop: enable Gitlab/BulkInsert
 end

app/views/admin/labels/edit.html.haml

@@ -1,4 +1,5 @@
 - add_to_breadcrumbs _("Labels"), admin_labels_path
+- add_to_breadcrumbs @label.name, admin_labels_path
 - breadcrumb_title _("Edit")
 - page_title _("Edit"), @label.name, _("Labels")
 %h1.page-title.gl-text-size-h-display

app/views/groups/labels/edit.html.haml

@@ -1,4 +1,5 @@
 - add_to_breadcrumbs _("Labels"), group_labels_path(@group)
+- add_to_breadcrumbs @label.name, group_labels_path(@group)
 - breadcrumb_title _("Edit")
 - page_title _("Edit"), @label.name, _("Labels")
 - show_lock_on_merge = @group.supports_lock_on_merge?

app/views/groups/milestones/edit.html.haml

@@ -1,7 +1,7 @@
 - breadcrumb_title _("Edit")
-- page_title _("Milestones")
-
-- render "header_title"
+- add_to_breadcrumbs _('Milestones'), group_milestones_path(@group)
+- add_to_breadcrumbs @milestone.title, milestone_path(@milestone)
+- page_title _('Edit'), @milestone.title, _('Milestones')
 
 %h1.page-title.gl-text-size-h-display
   = _('Edit milestone')

app/views/projects/labels/edit.html.haml

@@ -1,4 +1,5 @@
 - add_to_breadcrumbs _("Labels"), project_labels_path(@project)
+- add_to_breadcrumbs @label.name, project_labels_path(@project)
 - breadcrumb_title _("Edit")
 - page_title _("Edit"), @label.name, _("Labels")
 - show_lock_on_merge = @project.supports_lock_on_merge?

app/views/projects/milestones/edit.html.haml

@@ -1,5 +1,6 @@
 - breadcrumb_title _('Edit')
 - add_to_breadcrumbs _('Milestones'), project_milestones_path(@project)
+- add_to_breadcrumbs @milestone.title, milestone_path(@milestone)
 - page_title _('Edit'), @milestone.title, _('Milestones')
 
 %h1.page-title.gl-text-size-h-display

db/docs/batched_background_migrations/backfill_packages_dependencies_project_id.yml

@@ -8,4 +8,4 @@ feature_category: package_registry
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/161798
 milestone: '17.4'
 queued_migration_version: 20240906080833
-finalized_by: # version of the migration that finalized this BBM
+finalized_by: 20241016070006

db/docs/packages_dependencies.yml

@@ -11,4 +11,5 @@ milestone: '12.6'
 gitlab_schema: gitlab_main_cell
 allow_cross_foreign_keys:
 - gitlab_main_clusterwide
-sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/465276
+sharding_key:
+  project_id: projects

db/post_migrate/20241016070006_finalize_backfill_packages_dependencies_project_id.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class FinalizeBackfillPackagesDependenciesProjectId < Gitlab::Database::Migration[2.2]
+  milestone '17.6'
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+  disable_ddl_transaction!
+
+  MIGRATION = 'BackfillPackagesDependenciesProjectId'
+
+  def up
+    ensure_batched_background_migration_is_finished(
+      job_class_name: MIGRATION,
+      table_name: :packages_dependency_links,
+      column_name: :id,
+      job_arguments: [],
+      finalize: true
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/post_migrate/20241016072342_add_not_null_constraint_to_packages_dependencies_project_id.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class AddNotNullConstraintToPackagesDependenciesProjectId < Gitlab::Database::Migration[2.2]
+  disable_ddl_transaction!
+  milestone '17.6'
+
+  def up
+    add_not_null_constraint :packages_dependencies, :project_id
+  end
+
+  def down
+    remove_not_null_constraint :packages_dependencies, :project_id
+  end
+end

db/schema_migrations/20241016070006

@@ -0,0 +1 @@
+6c98f98c87902e68b3e00982c82c940c0f2313ab65677da982eebb74a1e718aa

db/schema_migrations/20241016072342

@@ -0,0 +1 @@
+9fb4adaac1e36188437e7d925ed0e6444fabc38d66a5730172af4244b1a69c93

db/structure.sql

@@ -15636,7 +15636,8 @@ CREATE TABLE packages_dependencies (
     id bigint NOT NULL,
     name character varying(255) NOT NULL,
     version_pattern character varying(255) NOT NULL,
-    project_id bigint
+    project_id bigint,
+    CONSTRAINT check_83faf1f5e7 CHECK ((project_id IS NOT NULL))
 );
 
 CREATE SEQUENCE packages_dependencies_id_seq

doc/.vale/gitlab_base/British.yml

@@ -8,6 +8,8 @@ extends: substitution
 message: "Use the US spelling '%s' instead of the British '%s'."
 link: https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#language
 level: error
+action:
+  name: replace
 ignorecase: true
 swap:
   aeon: eon

doc/.vale/gitlab_base/SubstitutionWarning.yml

@@ -9,6 +9,8 @@ extends: substitution
 message: "Use '%s' instead of '%s' when possible."
 link: https://docs.gitlab.com/ee/development/documentation/styleguide/word_list.html
 level: warning
+action:
+  name: replace
 ignorecase: true
 swap:
   active user: "billable user"

doc/.vale/gitlab_base/Substitutions.yml

@@ -9,6 +9,8 @@ extends: substitution
 message: "Use '%s' instead of '%s'."
 link: https://handbook.gitlab.com/handbook/communication/top-misused-terms/
 level: error
+action:
+  name: replace
 ignorecase: true
 swap:
   admin user: administrator

doc/.vale/gitlab_base/ToDo.yml

@@ -8,6 +8,8 @@ extends: substitution
 message: "Use 'to-do item' in most cases, or 'Add a to do' if referring to the UI button."
 link: https://docs.gitlab.com/ee/development/documentation/styleguide/word_list.html#to-do-item
 level: warning
+action:
+  name: replace
 ignorecase: false
 swap:
   '[Tt]o [Dd]o [Ii]tems?': to-do item

doc/administration/gitaly/troubleshooting.md

@@ -333,7 +333,7 @@ You might see the following in Gitaly and Praefect logs:
 ```shell
 {
   ...
-  "error":"rpc error: code = PermissionDenied desc = permission denied",
+  "error":"rpc error: code = PermissionDenied desc = permission denied: token has expired",
   "grpc.code":"PermissionDenied",
   "grpc.meta.client_name":"gitlab-web",
   "grpc.request.fullMethod":"/gitaly.ServerService/ServerInfo",
@@ -362,16 +362,6 @@ continue to listen on the old address after a `sudo gitlab-ctl reconfigure`.
 When this occurs, run `sudo gitlab-ctl restart` to resolve the issue. This should no longer be
 necessary because [this issue](https://gitlab.com/gitlab-org/gitaly/-/issues/2521) is resolved.
 
-## Errors in Gitaly logs when accessing repositories from a standalone Gitaly node
-
-You might see permission-denied errors in the Gitaly logs when you access a repository
-from a standalone Gitaly node. This error occurs even though file permissions are correct.
-It's likely that the Gitaly node is
-experiencing [clock drift](https://en.wikipedia.org/wiki/Clock_drift).
-
-Ensure that the GitLab and Gitaly nodes are synchronized and use an NTP time
-server to keep them synchronized if possible.
-
 ## Health check warnings
 
 The following warning in `/var/log/gitlab/praefect/current` can be ignored.

doc/administration/gitaly/troubleshooting_gitaly_cluster.md

@@ -80,28 +80,6 @@ If this check fails:
 1. Check if there is a high load on the Praefect database. If the Praefect database is slow to respond, it can lead health checks failing to persist
    to the database, leading Praefect to think nodes are unhealthy.
 
-### Check clock synchronization
-
-Authentication between Praefect and the Gitaly servers requires the server times to be
-within 60 seconds of each other, so that the token check succeeds.
-
-This check helps identify the root cause of `permission denied`
-[errors being logged by Praefect](troubleshooting.md#permission-denied-errors-appearing-in-gitaly-or-praefect-logs-when-accessing-repositories).
-
-For offline environments where access to public `pool.ntp.org` servers is not possible, the Praefect `check` sub-command fails this
-check with an error message similar to:
-
-```plaintext
-checking with NTP service at  and allowed clock drift 60000ms [correlation_id: <XXX>]
-Failed (fatal) error: gitaly node at tcp://[gitlab.example-instance.com]:8075: rpc error: code = DeadlineExceeded desc = context deadline exceeded
-```
-
-To resolve this issue, set an environment variable on all Praefect servers to point to an accessible internal [Network Time Protocol](https://en.wikipedia.org/wiki/Network_Time_Protocol) (NTP) server. For example:
-
-```shell
-export NTP_HOST=ntp.example.com
-```
-
 ## Praefect errors in logs
 
 If you receive an error, check `/var/log/gitlab/gitlab-rails/production.log`.

doc/development/migration_style_guide.md

@@ -196,6 +196,14 @@ VERSION=<migration ID> bundle exec rails db:migrate:main
 
 After a table has been created, it should be added to the database dictionary, following the steps mentioned in the [database dictionary guide](database/database_dictionary.md#adding-tables).
 
+### Migration checksum
+
+When a migration is first executed, a new file is created in [db/schema_migrations](https://gitlab.com/gitlab-org/gitlab/-/tree/v17.5.0-ee/db/schema_migrations) containing a `SHA` generated from the migration's timestamp. The name of this new file is the same as the [timestamp portion](#migration-timestamp-age) of the migration filename, for example [db/schema_migrations/20241021120146](https://gitlab.com/gitlab-org/gitlab/blob/aa7cfb42c312/db/schema_migrations/20241021120146).
+
+This new `db/schema_migration/<timestamp>` file indicates that the migration was executed successfuly and the result recorded in `db/structure.sql`. The presence of this file prevents the same migration from being executed twice, and therefore, it's necessary to include this file in the merge request that adds the new migration.
+
+See [Development change: Database schema version handling outside of structure.sql](https://gitlab.com/gitlab-org/gitlab/-/issues/218590) for more details about the `db/schema_migrations` directory.
+
 ## Avoiding downtime
 
 The document ["Avoiding downtime in migrations"](database/avoiding_downtime_in_migrations.md) specifies

doc/development/sidekiq/index.md

@@ -209,6 +209,9 @@ To determine an appropriate limit, you can use this PromQL query as a guide in [
 )
 ```
 
+NOTE:
+The [concurrency limit may be momentarily exceeded](https://gitlab.com/gitlab-org/gitlab/-/issues/490936#note_2172737349) and should not be relied on as a strict limit.
+
 ## Deferring Sidekiq workers
 
 Sidekiq workers are deferred by two ways,

doc/install/aws/index.md

@@ -63,7 +63,7 @@ It can take a few hours to validate a certificate provisioned through ACM. To av
 
 Below is a diagram of the recommended architecture.
 
-![AWS architecture diagram](img/aws_ha_architecture_diagram_v17_0.png)
+![Scaled down 2 Availability Zone Non-HA AWS architecture](img/aws_ha_architecture_diagram_v17_0.png)
 
 ## AWS costs
 

locale/gitlab.pot

@@ -49717,6 +49717,9 @@ msgstr ""
 msgid "SecurityOrchestration|Policy exceeds the maximum of %{limit} actions"
 msgstr ""
 
+msgid "SecurityOrchestration|Policy has reached the maximum of %{actionsCount} %{actions}"
+msgstr ""
+
 msgid "SecurityOrchestration|Policy is invalid"
 msgstr ""
 
@@ -64169,6 +64172,11 @@ msgstr[1] ""
 msgid "access:"
 msgstr ""
 
+msgid "action"
+msgid_plural "actions"
+msgstr[0] ""
+msgstr[1] ""
+
 msgid "active project"
 msgstr ""
 

spec/models/packages/dependency_spec.rb

@@ -20,6 +20,7 @@ RSpec.describe Packages::Dependency, type: :model, feature_category: :package_re
 
     it { is_expected.to validate_presence_of(:name) }
     it { is_expected.to validate_presence_of(:version_pattern) }
+    it { is_expected.to validate_presence_of(:project_id) }
 
     context 'uniqueness' do
       let_it_be(:project) { create(:project) }
@@ -63,17 +64,13 @@ RSpec.describe Packages::Dependency, type: :model, feature_category: :package_re
       create(:packages_dependency, name: 'foo', version_pattern: '~1.0.0', project: project)
     end
 
-    let_it_be(:package_dependency2) do
-      create(:packages_dependency, name: 'bar', version_pattern: '~2.5.0', project: nil)
-    end
-
     let_it_be(:package_dependency_diff_project) do
       create(:packages_dependency, name: 'bar', version_pattern: '~2.5.0', project: project2)
     end
 
-    let_it_be(:expected_ids) { [package_dependency1.id, package_dependency2.id] }
+    let_it_be(:expected_ids) { [package_dependency1.id] }
 
-    let(:names_and_version_patterns) { build_names_and_version_patterns(package_dependency1, package_dependency2) }
+    let(:names_and_version_patterns) { build_names_and_version_patterns(package_dependency1) }
     let(:chunk_size) { 50 }
     let(:rows_limit) { 50 }
 
@@ -103,7 +100,7 @@ RSpec.describe Packages::Dependency, type: :model, feature_category: :package_re
     context 'with a name bigger than column size' do
       let_it_be(:big_name) { 'a' * (Packages::Dependency::MAX_STRING_LENGTH + 1) }
 
-      let(:names_and_version_patterns) { build_names_and_version_patterns(package_dependency1, package_dependency2).merge(big_name => '~1.0.0') }
+      let(:names_and_version_patterns) { build_names_and_version_patterns(package_dependency1).merge(big_name => '~1.0.0') }
 
       it { is_expected.to match_array(expected_ids) }
     end
@@ -111,7 +108,7 @@ RSpec.describe Packages::Dependency, type: :model, feature_category: :package_re
     context 'with a version pattern bigger than column size' do
       let_it_be(:big_version_pattern) { 'a' * (Packages::Dependency::MAX_STRING_LENGTH + 1) }
 
-      let(:names_and_version_patterns) { build_names_and_version_patterns(package_dependency1, package_dependency2).merge('test' => big_version_pattern) }
+      let(:names_and_version_patterns) { build_names_and_version_patterns(package_dependency1).merge('test' => big_version_pattern) }
 
       it { is_expected.to match_array(expected_ids) }
     end
@@ -124,28 +121,28 @@ RSpec.describe Packages::Dependency, type: :model, feature_category: :package_re
     end
 
     context 'with parameters size' do
-      let_it_be(:package_dependency3) do
+      let_it_be(:package_dependency2) do
         create(:packages_dependency, name: 'foo3', version_pattern: '~1.5.3', project: project)
       end
 
-      let_it_be(:package_dependency4) do
+      let_it_be(:package_dependency3) do
         create(:packages_dependency, name: 'foo4', version_pattern: '~1.5.4', project: project)
       end
 
-      let_it_be(:package_dependency5) do
+      let_it_be(:package_dependency4) do
         create(:packages_dependency, name: 'foo5', version_pattern: '~1.5.5', project: project)
       end
 
-      let_it_be(:package_dependency6) do
+      let_it_be(:package_dependency5) do
         create(:packages_dependency, name: 'foo6', version_pattern: '~1.5.6', project: project)
       end
 
-      let_it_be(:package_dependency7) do
+      let_it_be(:package_dependency6) do
         create(:packages_dependency, name: 'foo7', version_pattern: '~1.5.7', project: project)
       end
 
-      let(:expected_ids) { [package_dependency1.id, package_dependency2.id, package_dependency3.id, package_dependency4.id, package_dependency5.id, package_dependency6.id, package_dependency7.id] }
-      let(:names_and_version_patterns) { build_names_and_version_patterns(package_dependency1, package_dependency2, package_dependency3, package_dependency4, package_dependency5, package_dependency6, package_dependency7) }
+      let(:expected_ids) { [package_dependency1.id, package_dependency2.id, package_dependency3.id, package_dependency4.id, package_dependency5.id, package_dependency6.id] }
+      let(:names_and_version_patterns) { build_names_and_version_patterns(package_dependency1, package_dependency2, package_dependency3, package_dependency4, package_dependency5, package_dependency6) }
 
       context 'above the chunk size' do
         let(:chunk_size) { 2 }
@@ -166,17 +163,13 @@ RSpec.describe Packages::Dependency, type: :model, feature_category: :package_re
       create(:packages_dependency, name: 'foo', version_pattern: '~1.0.0', project: project)
     end
 
-    let_it_be(:package_dependency2) do
-      create(:packages_dependency, name: 'bar', version_pattern: '~2.5.0', project: nil)
-    end
-
     let_it_be(:package_dependency_diff_project) do
       create(:packages_dependency, name: 'bar', version_pattern: '~2.5.0', project: project2)
     end
 
-    let_it_be(:expected_array) { [package_dependency1, package_dependency2] }
+    let_it_be(:expected_array) { [package_dependency1] }
 
-    let(:names_and_version_patterns) { build_names_and_version_patterns(package_dependency1, package_dependency2) }
+    let(:names_and_version_patterns) { build_names_and_version_patterns(package_dependency1) }
 
     subject do
       described_class.for_package_project_id_names_and_version_patterns(project.id, names_and_version_patterns)
@@ -210,17 +203,6 @@ RSpec.describe Packages::Dependency, type: :model, feature_category: :package_re
     end
   end
 
-  # TODO: remove the update operation when all packages dependencies have a `project_id`.
-  # https://gitlab.com/gitlab-org/gitlab/-/issues/481541
-  describe '.without_project' do
-    let_it_be(:dependency_1) { create(:packages_dependency, project: nil) }
-    let_it_be(:dependency_2) { create(:packages_dependency) }
-
-    it 'returns dependency records without project' do
-      expect(described_class.without_project).to contain_exactly(dependency_1)
-    end
-  end
-
   def build_names_and_version_patterns(*package_dependencies)
     result = Hash.new { |h, dependency| h[dependency.name] = dependency.version_pattern }
     package_dependencies.each { |dependency| result[dependency] }

spec/services/packages/create_dependency_service_spec.rb

@@ -115,20 +115,6 @@ RSpec.describe Packages::CreateDependencyService, feature_category: :package_reg
             end
           end
         end
-
-        # TODO: remove the test when all packages dependencies have a `project_id`.
-        # https://gitlab.com/gitlab-org/gitlab/-/issues/481541
-        context 'without project' do
-          let(:project) { nil }
-
-          it_behaves_like 'reuses dependencies'
-
-          it 'updates the project for existing dependency' do
-            subject
-
-            expect(dependency.reload.project_id).to eq(package.project_id)
-          end
-        end
       end
 
       context 'with a dependency not described with a hash' do