From b57bbf5a40379ae778e1db44cd6a9a62dd0fd1f5 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 2 Apr 2025 06:18:45 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- app/models/merge_request.rb | 4 -- config/gitlab_loose_foreign_keys.yml | 4 ++ ...ected_environment_deploy_access_levels.yml | 23 ++---------- ...add_ci_build_id_to_deployment_approvals.rb | 9 +++++ ...dex_to_deployment_approvals_ci_build_id.rb | 18 +++++++++ ...oy_access_levels_sharding_keys_not_null.rb | 22 +++++++++++ db/schema_migrations/20250319030730 | 1 + db/schema_migrations/20250326040902 | 1 + db/schema_migrations/20250328040632 | 1 + db/structure.sql | 4 ++ ...ported_models_and_hardware_requirements.md | 10 ++--- .../database/batched_background_migrations.md | 37 +++++++++++++++++++ doc/update/zero_downtime.md | 2 +- lib/gitlab/ci/templates/Ruby.gitlab-ci.yml | 6 ++- locale/gitlab.pot | 30 ++++++++++++++- spec/models/merge_request_spec.rb | 9 ----- 16 files changed, 139 insertions(+), 42 deletions(-) create mode 100644 db/migrate/20250326040902_add_ci_build_id_to_deployment_approvals.rb create mode 100644 db/migrate/20250328040632_add_index_to_deployment_approvals_ci_build_id.rb create mode 100644 db/post_migrate/20250319030730_add_protected_environment_deploy_access_levels_sharding_keys_not_null.rb create mode 100644 db/schema_migrations/20250319030730 create mode 100644 db/schema_migrations/20250326040902 create mode 100644 db/schema_migrations/20250328040632 diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 07dc16e8b5e..23abe167203 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -2413,10 +2413,6 @@ class MergeRequest < ApplicationRecord prepared_at.present? end - def prepare - NewMergeRequestWorker.perform_async(id, author_id) - end - def check_for_spam?(*) spammable_attribute_changed? && project.public? end diff --git a/config/gitlab_loose_foreign_keys.yml b/config/gitlab_loose_foreign_keys.yml index 61ca17de622..bc3481a5102 100644 --- a/config/gitlab_loose_foreign_keys.yml +++ b/config/gitlab_loose_foreign_keys.yml @@ -300,6 +300,10 @@ dependency_list_exports: - table: organizations column: organization_id on_delete: async_delete +deployment_approvals: + - table: ci_builds + column: ci_build_id + on_delete: async_delete deployment_clusters: - table: clusters column: cluster_id diff --git a/db/docs/protected_environment_deploy_access_levels.yml b/db/docs/protected_environment_deploy_access_levels.yml index e26c13fc00d..d68f473d955 100644 --- a/db/docs/protected_environment_deploy_access_levels.yml +++ b/db/docs/protected_environment_deploy_access_levels.yml @@ -10,24 +10,7 @@ description: A rule associated to a protected environment that allows a user, gr introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6672 milestone: '11.3' gitlab_schema: gitlab_main_cell -desired_sharding_key: - protected_environment_project_id: - references: projects - backfill_via: - parent: - foreign_key: protected_environment_id - table: protected_environments - sharding_key: project_id - belongs_to: protected_environment - protected_environment_group_id: - references: namespaces - backfill_via: - parent: - foreign_key: protected_environment_id - table: protected_environments - sharding_key: group_id - belongs_to: protected_environment -desired_sharding_key_migration_job_name: -- BackfillProtectedEnvironmentDeployAccessLevelsProtectedEnvironmentProjectId -- BackfillProtectedEnvironmentDeployAccessLevelsProtectedEnvironmentGroupId +sharding_key: + protected_environment_project_id: projects + protected_environment_group_id: namespaces table_size: small diff --git a/db/migrate/20250326040902_add_ci_build_id_to_deployment_approvals.rb b/db/migrate/20250326040902_add_ci_build_id_to_deployment_approvals.rb new file mode 100644 index 00000000000..5d6bf90da05 --- /dev/null +++ b/db/migrate/20250326040902_add_ci_build_id_to_deployment_approvals.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddCiBuildIdToDeploymentApprovals < Gitlab::Database::Migration[2.2] + milestone '17.11' + + def change + add_column :deployment_approvals, :ci_build_id, :bigint + end +end diff --git a/db/migrate/20250328040632_add_index_to_deployment_approvals_ci_build_id.rb b/db/migrate/20250328040632_add_index_to_deployment_approvals_ci_build_id.rb new file mode 100644 index 00000000000..e81b3ab0f8f --- /dev/null +++ b/db/migrate/20250328040632_add_index_to_deployment_approvals_ci_build_id.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddIndexToDeploymentApprovalsCiBuildId < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + + milestone '17.11' + + TABLE_NAME = :deployment_approvals + INDEX_NAME = 'index_deployment_approvals_on_ci_build_id' + + def up + add_concurrent_index TABLE_NAME, :ci_build_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name TABLE_NAME, INDEX_NAME + end +end diff --git a/db/post_migrate/20250319030730_add_protected_environment_deploy_access_levels_sharding_keys_not_null.rb b/db/post_migrate/20250319030730_add_protected_environment_deploy_access_levels_sharding_keys_not_null.rb new file mode 100644 index 00000000000..5b793a80078 --- /dev/null +++ b/db/post_migrate/20250319030730_add_protected_environment_deploy_access_levels_sharding_keys_not_null.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +class AddProtectedEnvironmentDeployAccessLevelsShardingKeysNotNull < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + milestone '17.11' + + def up + add_multi_column_not_null_constraint( + :protected_environment_deploy_access_levels, + :protected_environment_project_id, + :protected_environment_group_id + ) + end + + def down + remove_multi_column_not_null_constraint( + :protected_environment_deploy_access_levels, + :protected_environment_project_id, + :protected_environment_group_id + ) + end +end diff --git a/db/schema_migrations/20250319030730 b/db/schema_migrations/20250319030730 new file mode 100644 index 00000000000..b5c3984a0d4 --- /dev/null +++ b/db/schema_migrations/20250319030730 @@ -0,0 +1 @@ +dd6ec587afb4b9bc5e642a87f005fff93d70e1ec6cb59e31bf6c96eb05c4e75a \ No newline at end of file diff --git a/db/schema_migrations/20250326040902 b/db/schema_migrations/20250326040902 new file mode 100644 index 00000000000..202ebfa9473 --- /dev/null +++ b/db/schema_migrations/20250326040902 @@ -0,0 +1 @@ +58b6d6a27543c7fc00b281750959fe90822af30fc9a0a710f1f7cd1da91deab3 \ No newline at end of file diff --git a/db/schema_migrations/20250328040632 b/db/schema_migrations/20250328040632 new file mode 100644 index 00000000000..f0136220039 --- /dev/null +++ b/db/schema_migrations/20250328040632 @@ -0,0 +1 @@ +da18350f04353273105f23d7610c307efbbec736d043a3c91920e440edde3b90 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 9f1763431ea..3375d7be6ee 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -13618,6 +13618,7 @@ CREATE TABLE deployment_approvals ( comment text, approval_rule_id bigint, project_id bigint, + ci_build_id bigint, CONSTRAINT check_692c1d1b90 CHECK ((project_id IS NOT NULL)), CONSTRAINT check_e2eb6a17d8 CHECK ((char_length(comment) <= 255)) ); @@ -21635,6 +21636,7 @@ CREATE TABLE protected_environment_deploy_access_levels ( group_inheritance_type smallint DEFAULT 0 NOT NULL, protected_environment_project_id bigint, protected_environment_group_id bigint, + CONSTRAINT check_cee712b465 CHECK ((num_nonnulls(protected_environment_group_id, protected_environment_project_id) = 1)), CONSTRAINT check_deploy_access_levels_user_group_access_level_any_not_null CHECK ((num_nonnulls(user_id, group_id, access_level) = 1)) ); @@ -34575,6 +34577,8 @@ CREATE UNIQUE INDEX index_deploy_tokens_on_token_encrypted ON deploy_tokens USIN CREATE INDEX index_deployment_approvals_on_approval_rule_id ON deployment_approvals USING btree (approval_rule_id); +CREATE INDEX index_deployment_approvals_on_ci_build_id ON deployment_approvals USING btree (ci_build_id); + CREATE INDEX index_deployment_approvals_on_created_at_and_id ON deployment_approvals USING btree (created_at, id); CREATE UNIQUE INDEX index_deployment_approvals_on_deployment_user_approval_rule ON deployment_approvals USING btree (deployment_id, user_id, approval_rule_id); diff --git a/doc/administration/gitlab_duo_self_hosted/supported_models_and_hardware_requirements.md b/doc/administration/gitlab_duo_self_hosted/supported_models_and_hardware_requirements.md index 23d919ab2b1..1397119c08d 100644 --- a/doc/administration/gitlab_duo_self_hosted/supported_models_and_hardware_requirements.md +++ b/doc/administration/gitlab_duo_self_hosted/supported_models_and_hardware_requirements.md @@ -57,11 +57,11 @@ The following models are configurable for the functionalities marked below, but | Model family | Model | Supported platforms | Status | Code completion | Code generation | GitLab Duo Chat | |--------------- |-------|---------------------|--------|-----------------|-----------------|-----------------| -| Llama | [Llama 3 8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) | [vLLM](supported_llm_serving_platforms.md#for-self-hosted-model-deployments), [AWS Bedrock](https://aws.amazon.com/bedrock/llama/) | Beta | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | -| Llama | [Llama 3.1 8B](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | [vLLM](supported_llm_serving_platforms.md#for-self-hosted-model-deployments), [AWS Bedrock](https://aws.amazon.com/bedrock/llama/) | Beta | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | -| Llama | [Llama 3 70B](https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct) | [vLLM](supported_llm_serving_platforms.md#for-self-hosted-model-deployments), [AWS Bedrock](https://aws.amazon.com/bedrock/llama/) | Beta | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | -| Llama | [Llama 3.1 70B](https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct) | [vLLM](supported_llm_serving_platforms.md#for-self-hosted-model-deployments), [AWS Bedrock](https://aws.amazon.com/bedrock/llama/) | Beta | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | -| Llama | [Llama 3.3 70B](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) | [vLLM](supported_llm_serving_platforms.md#for-self-hosted-model-deployments), [AWS Bedrock](https://aws.amazon.com/bedrock/llama/) | Beta | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | +| Llama | [Llama 3 8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) | [vLLM](supported_llm_serving_platforms.md#for-self-hosted-model-deployments) | Beta | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | +| Llama | [Llama 3.1 8B](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | [vLLM](supported_llm_serving_platforms.md#for-self-hosted-model-deployments) | Beta | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | +| Llama | [Llama 3 70B](https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct) | [vLLM](supported_llm_serving_platforms.md#for-self-hosted-model-deployments) | Beta | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | +| Llama | [Llama 3.1 70B](https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct) | [vLLM](supported_llm_serving_platforms.md#for-self-hosted-model-deployments) | Beta | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | +| Llama | [Llama 3.3 70B](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct) | [vLLM](supported_llm_serving_platforms.md#for-self-hosted-model-deployments) | Beta | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | {{< icon name="check-circle" >}} Yes | | CodeGemma | [CodeGemma 2b](https://huggingface.co/google/codegemma-2b) | [vLLM](supported_llm_serving_platforms.md#for-self-hosted-model-deployments) | Experimental | {{< icon name="check-circle" >}} Yes | {{< icon name="dotted-circle" >}} No | {{< icon name="dotted-circle" >}} No | | CodeGemma | [CodeGemma 7b-it](https://huggingface.co/google/codegemma-7b-it) | [vLLM](supported_llm_serving_platforms.md#for-self-hosted-model-deployments) | Experimental | {{< icon name="dotted-circle" >}} No | {{< icon name="check-circle" >}} Yes | {{< icon name="dotted-circle" >}} No | | CodeGemma | [CodeGemma 7b-code](https://huggingface.co/google/codegemma-7b) | [vLLM](supported_llm_serving_platforms.md#for-self-hosted-model-deployments) | Experimental | {{< icon name="check-circle" >}} Yes | {{< icon name="dotted-circle" >}} No | {{< icon name="dotted-circle" >}} No | diff --git a/doc/development/database/batched_background_migrations.md b/doc/development/database/batched_background_migrations.md index d3a6cb996a5..5088e2899a7 100644 --- a/doc/development/database/batched_background_migrations.md +++ b/doc/development/database/batched_background_migrations.md @@ -1189,6 +1189,43 @@ for more details. end ``` +1. If possible update the entire sub-batch in a single query + instead of updating each model separately. + This can be achieve in different ways, depending on the scenario. + + - Generate an `UPDATE` query, and use `FROM` to join the tables + that provide the necessary values + ([example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/184051)). + - Generate an `UPDATE` query, and use `FROM(VALUES( ...))` to + pass values calculated beforehand + ([example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/177993)). + - Pass all keys and values to `ActiveRelation#update`. + + ```ruby + # good + def perform + each_sub_batch do |sub_batch| + connection.execute <<~SQL + UPDATE fork_networks + SET organization_id = projects.organization_id + FROM projects + WHERE fork_networks.id IN (#{sub_batch.pluck(:id)}) + AND fork_networks.root_project_id = projects.id + AND fork_networks.organization_id IS NULL + SQL + end + end + + # bad + def perform + each_sub_batch do |sub_batch| + sub_batch.each |fork_network| + fork_network.update!(organization_id: fork_network.root_project.organization_id) + end + end + end + ``` + ## Examples ### Routes use-case diff --git a/doc/update/zero_downtime.md b/doc/update/zero_downtime.md index 28f19383ceb..d3c2da5999e 100644 --- a/doc/update/zero_downtime.md +++ b/doc/update/zero_downtime.md @@ -264,7 +264,7 @@ In addition to the above, Rails is where the main database migrations need to be 1. [Upgrade the GitLab package](package/_index.md#upgrade-to-a-specific-version). - 1. Configure regular migrations to by setting `gitlab_rails['auto_migrate'] = true` in the + 1. Configure regular migrations to run by setting `gitlab_rails['auto_migrate'] = true` in the `/etc/gitlab/gitlab.rb` configuration file. - If the deploy node is currently going through PgBouncer to reach the database then you must [bypass it](../administration/postgresql/pgbouncer.md#procedure-for-bypassing-pgbouncer) diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 4f7bb28594e..54f45bf05fc 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -23,6 +23,9 @@ variables: # Cache gems in between builds cache: + key: + files: + - Gemfile.lock paths: - vendor/ruby @@ -32,7 +35,8 @@ before_script: - ruby -v # Print out ruby version for debugging # Uncomment next line if your rails app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - - bundle config set --local deployment true # Install dependencies into ./vendor/ruby + - bundle config set --local deployment true + - bundle config set --local path './vendor/ruby' # Install dependencies into ./vendor/ruby - bundle install -j $(nproc) # Optional - Delete if not using `rubocop` diff --git a/locale/gitlab.pot b/locale/gitlab.pot index cf0eb2382c1..6fc080130ff 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -3885,9 +3885,15 @@ msgstr "" msgid "AdherenceReport|Learn more about the changes in our %{linkStart}documentation%{linkEnd}." msgstr "" +msgid "AdherenceReport|No statuses found." +msgstr "" + msgid "AdherenceReport|Show old report" msgstr "" +msgid "AdherenceReport|There was an error loading adherence report." +msgstr "" + msgid "AdherenceReport|We've updated the adherence report with new features to enhance your compliance workflow." msgstr "" @@ -15560,10 +15566,15 @@ msgstr "" msgid "ComplianceReport|and %{count} more" msgstr "" -msgid "ComplianceStandardsAdherenceV2|New Report Placeholder" +msgid "ComplianceStandardsAdherence| Standards adherence export" msgstr "" -msgid "ComplianceStandardsAdherence| Standards adherence export" +msgid "ComplianceStandardsAdherence|%d control" +msgid_plural "ComplianceStandardsAdherence|%d controls" +msgstr[0] "" +msgstr[1] "" + +msgid "ComplianceStandardsAdherence|%{failedCount}/%{totalCount} failed" msgstr "" msgid "ComplianceStandardsAdherence|A rule is configured to prevent author approved merge requests." @@ -15620,6 +15631,12 @@ msgstr "" msgid "ComplianceStandardsAdherence|Filter by" msgstr "" +msgid "ComplianceStandardsAdherence|Fix suggestions" +msgstr "" + +msgid "ComplianceStandardsAdherence|Framework" +msgstr "" + msgid "ComplianceStandardsAdherence|Group by" msgstr "" @@ -15644,6 +15661,9 @@ msgstr "" msgid "ComplianceStandardsAdherence|How to fix" msgstr "" +msgid "ComplianceStandardsAdherence|Last scanned" +msgstr "" + msgid "ComplianceStandardsAdherence|Merge request approval rules" msgstr "" @@ -15671,6 +15691,12 @@ msgstr "" msgid "ComplianceStandardsAdherence|Other compliance frameworks applied to %{linkStart}%{projectName}%{linkEnd}" msgstr "" +msgid "ComplianceStandardsAdherence|Passed" +msgstr "" + +msgid "ComplianceStandardsAdherence|Pending" +msgstr "" + msgid "ComplianceStandardsAdherence|Prevent authors as approvers" msgstr "" diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb index dc20531722c..411c2dcc22a 100644 --- a/spec/models/merge_request_spec.rb +++ b/spec/models/merge_request_spec.rb @@ -6506,15 +6506,6 @@ RSpec.describe MergeRequest, factory_default: :keep, feature_category: :code_rev end end - describe 'prepare' do - it 'calls NewMergeRequestWorker' do - expect(NewMergeRequestWorker).to receive(:perform_async) - .with(subject.id, subject.author_id) - - subject.prepare - end - end - describe '#check_for_spam?' do let_it_be(:project) { create(:project, :public) } let(:merge_request) { build_stubbed(:merge_request, source_project: project) }