From bb4eeef95a7a5f01653bd3e3d8459c98e8a0e880 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 30 Apr 2024 03:11:32 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- ...cense_compliance_expand_failed_monthly.yml | 2 +- ...5305_i_quickactions_set_parent_monthly.yml | 0 ...ll_work_item_parent_links_namespace_id.yml | 9 ++ db/docs/dependency_list_exports.yml | 1 + db/docs/sbom_component_versions.yml | 1 + db/docs/sbom_components.yml | 1 + db/docs/sbom_source_packages.yml | 1 + db/docs/sbom_sources.yml | 1 + db/docs/security_findings.yml | 10 +- db/docs/security_scans.yml | 1 + db/docs/security_training_providers.yml | 1 + db/docs/vulnerability_exports.yml | 4 +- db/docs/work_item_parent_links.yml | 1 + ..._namespace_id_to_work_item_parent_links.rb | 9 ++ ..._work_item_parent_links_on_namespace_id.rb | 16 ++++ ..._work_item_parent_links_namespace_id_fk.rb | 16 ++++ ..._item_parent_links_namespace_id_trigger.rb | 25 +++++ ...ill_work_item_parent_links_namespace_id.rb | 40 ++++++++ ...s_on_target_project_id_and_locked_state.rb | 17 ++++ ...t_metrics_pipeline_id_convert_to_bigint.rb | 1 + db/schema_migrations/20240419035504 | 1 + db/schema_migrations/20240419035505 | 1 + db/schema_migrations/20240419035506 | 1 + db/schema_migrations/20240419035507 | 1 + db/schema_migrations/20240419035508 | 1 + db/schema_migrations/20240423022641 | 1 + db/structure.sql | 28 +++++- ...ill_work_item_parent_links_namespace_id.rb | 12 +++ lib/gitlab/gitaly_client/operation_service.rb | 15 +++ locale/gitlab.pot | 6 +- ...ork_item_parent_links_namespace_id_spec.rb | 15 +++ spec/lib/gitlab/database/sharding_key_spec.rb | 3 +- .../gitaly_client/operation_service_spec.rb | 93 ++++++++++++++----- ...ork_item_parent_links_namespace_id_spec.rb | 33 +++++++ 34 files changed, 322 insertions(+), 46 deletions(-) rename config/metrics/{counts_7d => counts_28d}/20230922165305_i_quickactions_set_parent_monthly.yml (100%) create mode 100644 db/docs/batched_background_migrations/backfill_work_item_parent_links_namespace_id.yml create mode 100644 db/migrate/20240419035504_add_namespace_id_to_work_item_parent_links.rb create mode 100644 db/migrate/20240419035505_index_work_item_parent_links_on_namespace_id.rb create mode 100644 db/migrate/20240419035506_add_work_item_parent_links_namespace_id_fk.rb create mode 100644 db/post_migrate/20240419035507_add_work_item_parent_links_namespace_id_trigger.rb create mode 100644 db/post_migrate/20240419035508_queue_backfill_work_item_parent_links_namespace_id.rb create mode 100644 db/post_migrate/20240423022641_drop_idx_merge_requests_on_target_project_id_and_locked_state.rb create mode 100644 db/schema_migrations/20240419035504 create mode 100644 db/schema_migrations/20240419035505 create mode 100644 db/schema_migrations/20240419035506 create mode 100644 db/schema_migrations/20240419035507 create mode 100644 db/schema_migrations/20240419035508 create mode 100644 db/schema_migrations/20240423022641 create mode 100644 lib/gitlab/background_migration/backfill_work_item_parent_links_namespace_id.rb create mode 100644 spec/lib/gitlab/background_migration/backfill_work_item_parent_links_namespace_id_spec.rb create mode 100644 spec/migrations/20240419035508_queue_backfill_work_item_parent_links_namespace_id_spec.rb diff --git a/config/metrics/counts_28d/20220830104528_i_code_review_merge_request_widget_license_compliance_expand_failed_monthly.yml b/config/metrics/counts_28d/20220830104528_i_code_review_merge_request_widget_license_compliance_expand_failed_monthly.yml index 75046f9f927..288961da080 100644 --- a/config/metrics/counts_28d/20220830104528_i_code_review_merge_request_widget_license_compliance_expand_failed_monthly.yml +++ b/config/metrics/counts_28d/20220830104528_i_code_review_merge_request_widget_license_compliance_expand_failed_monthly.yml @@ -1,6 +1,6 @@ --- key_path: redis_hll_counters.code_review.i_code_review_merge_request_widget_license_compliance_expand_failed_monthly -description: The count of unique users (monthly) who expanded the License Compliance widget extension while it is in its Failed state +description: The count of unique users (monthly) who expanded the License Compliance widget extension while it is in its Failed state. product_section: dev product_stage: create product_group: code_review diff --git a/config/metrics/counts_7d/20230922165305_i_quickactions_set_parent_monthly.yml b/config/metrics/counts_28d/20230922165305_i_quickactions_set_parent_monthly.yml similarity index 100% rename from config/metrics/counts_7d/20230922165305_i_quickactions_set_parent_monthly.yml rename to config/metrics/counts_28d/20230922165305_i_quickactions_set_parent_monthly.yml diff --git a/db/docs/batched_background_migrations/backfill_work_item_parent_links_namespace_id.yml b/db/docs/batched_background_migrations/backfill_work_item_parent_links_namespace_id.yml new file mode 100644 index 00000000000..3a0564b63ca --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_work_item_parent_links_namespace_id.yml @@ -0,0 +1,9 @@ +--- +migration_job_name: BackfillWorkItemParentLinksNamespaceId +description: Backfills sharding key `work_item_parent_links.namespace_id` from `issues`. +feature_category: team_planning +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/150075 +milestone: '17.0' +queued_migration_version: 20240419035508 +finalize_after: '2024-05-22' +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/dependency_list_exports.yml b/db/docs/dependency_list_exports.yml index 1597f17c80c..c60c3aa4dbc 100644 --- a/db/docs/dependency_list_exports.yml +++ b/db/docs/dependency_list_exports.yml @@ -8,3 +8,4 @@ description: Dependency list exported data introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104361 milestone: '15.7' gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/454947 diff --git a/db/docs/sbom_component_versions.yml b/db/docs/sbom_component_versions.yml index 25ef9e3fb9c..ab548f9dc24 100644 --- a/db/docs/sbom_component_versions.yml +++ b/db/docs/sbom_component_versions.yml @@ -8,3 +8,4 @@ description: Stores version information for software components produced by a So introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90809 milestone: '15.2' gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/457096 diff --git a/db/docs/sbom_components.yml b/db/docs/sbom_components.yml index b735d7f29c2..4c22d906966 100644 --- a/db/docs/sbom_components.yml +++ b/db/docs/sbom_components.yml @@ -8,3 +8,4 @@ description: Stores information about software components produced by a Software introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90809 milestone: '15.2' gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/457096 diff --git a/db/docs/sbom_source_packages.yml b/db/docs/sbom_source_packages.yml index 2c0df1df88e..228bf34f28c 100644 --- a/db/docs/sbom_source_packages.yml +++ b/db/docs/sbom_source_packages.yml @@ -8,3 +8,4 @@ description: Tracks Source Package of an SBOM Occurrence introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/140539 milestone: '16.8' gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/457096 diff --git a/db/docs/sbom_sources.yml b/db/docs/sbom_sources.yml index b9cc8611b45..e9621b6d895 100644 --- a/db/docs/sbom_sources.yml +++ b/db/docs/sbom_sources.yml @@ -8,3 +8,4 @@ description: Stores information about where an SBoM component originated from introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90812 milestone: '15.2' gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/457096 diff --git a/db/docs/security_findings.yml b/db/docs/security_findings.yml index de091dd1d00..b39d69b9744 100644 --- a/db/docs/security_findings.yml +++ b/db/docs/security_findings.yml @@ -14,12 +14,4 @@ allow_cross_transactions: - gitlab_main_clusterwide allow_cross_foreign_keys: - gitlab_main_clusterwide -desired_sharding_key: - project_id: - references: projects - backfill_via: - parent: - foreign_key: scanner_id - table: vulnerability_scanners - sharding_key: project_id - belongs_to: scanner +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/454949 diff --git a/db/docs/security_scans.yml b/db/docs/security_scans.yml index b89faf6584d..ce73f46afe3 100644 --- a/db/docs/security_scans.yml +++ b/db/docs/security_scans.yml @@ -8,3 +8,4 @@ description: Stores information about the security scans that are a part of Ci:: introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23669 milestone: '12.8' gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/454948 diff --git a/db/docs/security_training_providers.yml b/db/docs/security_training_providers.yml index 69c42a39d1a..53f9a609e49 100644 --- a/db/docs/security_training_providers.yml +++ b/db/docs/security_training_providers.yml @@ -8,3 +8,4 @@ description: Stores information about the available security training providers introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78195 milestone: '14.7' gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/458309 diff --git a/db/docs/vulnerability_exports.yml b/db/docs/vulnerability_exports.yml index 4a975fa67f5..537566ed635 100644 --- a/db/docs/vulnerability_exports.yml +++ b/db/docs/vulnerability_exports.yml @@ -14,6 +14,4 @@ allow_cross_transactions: - gitlab_main_clusterwide allow_cross_foreign_keys: - gitlab_main_clusterwide -sharding_key: - project_id: projects - group_id: namespaces +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/457095 diff --git a/db/docs/work_item_parent_links.yml b/db/docs/work_item_parent_links.yml index 185ffe712c9..2b6d324218a 100644 --- a/db/docs/work_item_parent_links.yml +++ b/db/docs/work_item_parent_links.yml @@ -23,3 +23,4 @@ desired_sharding_key: table: issues sharding_key: namespace_id belongs_to: work_item +desired_sharding_key_migration_job_name: BackfillWorkItemParentLinksNamespaceId diff --git a/db/migrate/20240419035504_add_namespace_id_to_work_item_parent_links.rb b/db/migrate/20240419035504_add_namespace_id_to_work_item_parent_links.rb new file mode 100644 index 00000000000..be9bd682b48 --- /dev/null +++ b/db/migrate/20240419035504_add_namespace_id_to_work_item_parent_links.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddNamespaceIdToWorkItemParentLinks < Gitlab::Database::Migration[2.2] + milestone '17.0' + + def change + add_column :work_item_parent_links, :namespace_id, :bigint + end +end diff --git a/db/migrate/20240419035505_index_work_item_parent_links_on_namespace_id.rb b/db/migrate/20240419035505_index_work_item_parent_links_on_namespace_id.rb new file mode 100644 index 00000000000..60414eddce1 --- /dev/null +++ b/db/migrate/20240419035505_index_work_item_parent_links_on_namespace_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexWorkItemParentLinksOnNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.0' + disable_ddl_transaction! + + INDEX_NAME = 'index_work_item_parent_links_on_namespace_id' + + def up + add_concurrent_index :work_item_parent_links, :namespace_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :work_item_parent_links, INDEX_NAME + end +end diff --git a/db/migrate/20240419035506_add_work_item_parent_links_namespace_id_fk.rb b/db/migrate/20240419035506_add_work_item_parent_links_namespace_id_fk.rb new file mode 100644 index 00000000000..59b27350962 --- /dev/null +++ b/db/migrate/20240419035506_add_work_item_parent_links_namespace_id_fk.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddWorkItemParentLinksNamespaceIdFk < Gitlab::Database::Migration[2.2] + milestone '17.0' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :work_item_parent_links, :namespaces, column: :namespace_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :work_item_parent_links, column: :namespace_id + end + end +end diff --git a/db/post_migrate/20240419035507_add_work_item_parent_links_namespace_id_trigger.rb b/db/post_migrate/20240419035507_add_work_item_parent_links_namespace_id_trigger.rb new file mode 100644 index 00000000000..ed309f704d3 --- /dev/null +++ b/db/post_migrate/20240419035507_add_work_item_parent_links_namespace_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddWorkItemParentLinksNamespaceIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.0' + + def up + install_sharding_key_assignment_trigger( + table: :work_item_parent_links, + sharding_key: :namespace_id, + parent_table: :issues, + parent_sharding_key: :namespace_id, + foreign_key: :work_item_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :work_item_parent_links, + sharding_key: :namespace_id, + parent_table: :issues, + parent_sharding_key: :namespace_id, + foreign_key: :work_item_id + ) + end +end diff --git a/db/post_migrate/20240419035508_queue_backfill_work_item_parent_links_namespace_id.rb b/db/post_migrate/20240419035508_queue_backfill_work_item_parent_links_namespace_id.rb new file mode 100644 index 00000000000..19d29c33852 --- /dev/null +++ b/db/post_migrate/20240419035508_queue_backfill_work_item_parent_links_namespace_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillWorkItemParentLinksNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.0' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillWorkItemParentLinksNamespaceId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :work_item_parent_links, + :id, + :namespace_id, + :issues, + :namespace_id, + :work_item_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :work_item_parent_links, + :id, + [ + :namespace_id, + :issues, + :namespace_id, + :work_item_id + ] + ) + end +end diff --git a/db/post_migrate/20240423022641_drop_idx_merge_requests_on_target_project_id_and_locked_state.rb b/db/post_migrate/20240423022641_drop_idx_merge_requests_on_target_project_id_and_locked_state.rb new file mode 100644 index 00000000000..66813ebc215 --- /dev/null +++ b/db/post_migrate/20240423022641_drop_idx_merge_requests_on_target_project_id_and_locked_state.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class DropIdxMergeRequestsOnTargetProjectIdAndLockedState < Gitlab::Database::Migration[2.2] + milestone '17.0' + + INDEX_NAME = 'idx_merge_requests_on_target_project_id_and_locked_state' + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name :merge_requests, name: INDEX_NAME + end + + def down + add_concurrent_index :merge_requests, :target_project_id, where: 'state_id = 4', name: INDEX_NAME + end +end diff --git a/db/post_migrate/20240424111535_swap_merge_request_metrics_pipeline_id_convert_to_bigint.rb b/db/post_migrate/20240424111535_swap_merge_request_metrics_pipeline_id_convert_to_bigint.rb index f50a6ab3b05..0ef7f00bfa2 100644 --- a/db/post_migrate/20240424111535_swap_merge_request_metrics_pipeline_id_convert_to_bigint.rb +++ b/db/post_migrate/20240424111535_swap_merge_request_metrics_pipeline_id_convert_to_bigint.rb @@ -35,6 +35,7 @@ class SwapMergeRequestMetricsPipelineIdConvertToBigint < Gitlab::Database::Migra swap_columns(TABLE_NAME, COLUMN_NAME, BIGINT_COLUMN_NAME) reset_trigger_function(:trigger_fd041fe2d1a7) + reset_trigger_function(:nullify_merge_request_metrics_build_data) # No defaults to swap as the column is not a PK one diff --git a/db/schema_migrations/20240419035504 b/db/schema_migrations/20240419035504 new file mode 100644 index 00000000000..91a5bff8df8 --- /dev/null +++ b/db/schema_migrations/20240419035504 @@ -0,0 +1 @@ +805d7a83dd75371722fd76bab301b988678fc5af2812330dc2f5f07cfa22d0b3 \ No newline at end of file diff --git a/db/schema_migrations/20240419035505 b/db/schema_migrations/20240419035505 new file mode 100644 index 00000000000..04306992928 --- /dev/null +++ b/db/schema_migrations/20240419035505 @@ -0,0 +1 @@ +19a099402fc483b94262e771e617bffc7189092692fc884022fc0578ce020b41 \ No newline at end of file diff --git a/db/schema_migrations/20240419035506 b/db/schema_migrations/20240419035506 new file mode 100644 index 00000000000..2b9b82d603a --- /dev/null +++ b/db/schema_migrations/20240419035506 @@ -0,0 +1 @@ +bce964a20bfd0c41f5e836f8366172e32461029f72b6d47b7f575c42b1511a07 \ No newline at end of file diff --git a/db/schema_migrations/20240419035507 b/db/schema_migrations/20240419035507 new file mode 100644 index 00000000000..c9306055da5 --- /dev/null +++ b/db/schema_migrations/20240419035507 @@ -0,0 +1 @@ +d479fd43927c47833dff923b3ba70dc262f663963ac6f91d2ca982ebfd033aa5 \ No newline at end of file diff --git a/db/schema_migrations/20240419035508 b/db/schema_migrations/20240419035508 new file mode 100644 index 00000000000..a5a2f1c444e --- /dev/null +++ b/db/schema_migrations/20240419035508 @@ -0,0 +1 @@ +6e8c7b47974742dd982082328ff915283c03829514c996e3db128bc9c651c38b \ No newline at end of file diff --git a/db/schema_migrations/20240423022641 b/db/schema_migrations/20240423022641 new file mode 100644 index 00000000000..798f25cd7f3 --- /dev/null +++ b/db/schema_migrations/20240423022641 @@ -0,0 +1 @@ +036a8b47c36e50b5c933722eb951c8f56fdcb088944cc3333e2797e43fe8fe7c \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 6b16a34c68b..76f689e8b96 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -729,6 +729,22 @@ BEGIN END; $$; +CREATE FUNCTION trigger_25c44c30884f() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."namespace_id" IS NULL THEN + SELECT "namespace_id" + INTO NEW."namespace_id" + FROM "issues" + WHERE "issues"."id" = NEW."work_item_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_2ac3d66ed1d3() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -18192,7 +18208,8 @@ CREATE TABLE work_item_parent_links ( work_item_parent_id bigint NOT NULL, relative_position integer, created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL + updated_at timestamp with time zone NOT NULL, + namespace_id bigint ); CREATE SEQUENCE work_item_parent_links_id_seq @@ -24207,8 +24224,6 @@ CREATE INDEX idx_merge_requests_on_merged_state ON merge_requests USING btree (i CREATE INDEX idx_merge_requests_on_source_project_and_branch_state_opened ON merge_requests USING btree (source_project_id, source_branch) WHERE (state_id = 1); -CREATE INDEX idx_merge_requests_on_target_project_id_and_locked_state ON merge_requests USING btree (target_project_id) WHERE (state_id = 4); - CREATE INDEX idx_merge_requests_on_unmerged_state_id ON merge_requests USING btree (id) WHERE (state_id <> 3); CREATE UNIQUE INDEX idx_metrics_users_starred_dashboard_on_user_project_dashboard ON metrics_users_starred_dashboards USING btree (user_id, project_id, dashboard_path); @@ -28005,6 +28020,8 @@ CREATE INDEX index_work_item_hierarchy_restrictions_on_parent_type_id ON work_it CREATE UNIQUE INDEX index_work_item_link_restrictions_on_source_link_type_target ON work_item_related_link_restrictions USING btree (source_type_id, link_type, target_type_id); +CREATE INDEX index_work_item_parent_links_on_namespace_id ON work_item_parent_links USING btree (namespace_id); + CREATE UNIQUE INDEX index_work_item_parent_links_on_work_item_id ON work_item_parent_links USING btree (work_item_id); CREATE INDEX index_work_item_parent_links_on_work_item_parent_id ON work_item_parent_links USING btree (work_item_parent_id); @@ -29901,6 +29918,8 @@ CREATE TRIGGER trigger_10ee1357e825 BEFORE INSERT OR UPDATE ON p_ci_builds FOR E CREATE TRIGGER trigger_2428b5519042 BEFORE INSERT OR UPDATE ON vulnerability_feedback FOR EACH ROW EXECUTE FUNCTION trigger_2428b5519042(); +CREATE TRIGGER trigger_25c44c30884f BEFORE INSERT OR UPDATE ON work_item_parent_links FOR EACH ROW EXECUTE FUNCTION trigger_25c44c30884f(); + CREATE TRIGGER trigger_2ac3d66ed1d3 BEFORE INSERT OR UPDATE ON vulnerability_occurrence_pipelines FOR EACH ROW EXECUTE FUNCTION trigger_2ac3d66ed1d3(); CREATE TRIGGER trigger_3857ca5ea4af BEFORE INSERT OR UPDATE ON merge_trains FOR EACH ROW EXECUTE FUNCTION trigger_3857ca5ea4af(); @@ -30655,6 +30674,9 @@ ALTER TABLE ONLY geo_event_log ALTER TABLE ONLY milestones ADD CONSTRAINT fk_9bd0a0c791 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; +ALTER TABLE ONLY work_item_parent_links + ADD CONSTRAINT fk_9be5ef5f80 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY issues ADD CONSTRAINT fk_9c4516d665 FOREIGN KEY (duplicated_to_id) REFERENCES issues(id) ON DELETE SET NULL; diff --git a/lib/gitlab/background_migration/backfill_work_item_parent_links_namespace_id.rb b/lib/gitlab/background_migration/backfill_work_item_parent_links_namespace_id.rb new file mode 100644 index 00000000000..e1c6bdaebba --- /dev/null +++ b/lib/gitlab/background_migration/backfill_work_item_parent_links_namespace_id.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Migration/BackgroundMigrationBaseClass -- BackfillDesiredShardingKeyJob inherits from BatchedMigrationJob. + class BackfillWorkItemParentLinksNamespaceId < BackfillDesiredShardingKeyJob + operation_name :backfill_work_item_parent_links_namespace_id + feature_category :team_planning + end + # rubocop: enable Migration/BackgroundMigrationBaseClass + end +end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index bc9512b1b5c..208774845b6 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -179,6 +179,7 @@ module Gitlab yield response_enum.next.commit_id request_enum.push(Gitaly::UserMergeBranchRequest.new(apply: true)) + request_enum.close second_response = response_enum.next @@ -186,6 +187,8 @@ module Gitlab return if branch_update.nil? raise Gitlab::Git::CommitError, 'failed to apply merge to branch' unless branch_update.commit_id.present? + consume_final_message(response_enum) + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) @@ -369,8 +372,11 @@ module Gitlab # Second request confirms with gitaly to finalize the rebase request_enum.push(Gitaly::UserRebaseConfirmableRequest.new(apply: true)) + request_enum.close response_enum.next + consume_final_message(response_enum) + rebase_sha rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) @@ -572,6 +578,15 @@ module Gitlab private + # consume_final_message consumes the final message that contains the status from the response + # stream and raises an exception if it wasn't the last one. + def consume_final_message(response_enum) + response_enum.next + rescue StopIteration + else + raise 'expected response stream to finish' + end + # rubocop:disable Metrics/ParameterLists def user_commit_files_request_header( user, branch_name, commit_message, actions, author_email, author_name, diff --git a/locale/gitlab.pot b/locale/gitlab.pot index e664f755c26..2a5240ae287 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -16152,6 +16152,9 @@ msgstr "" msgid "DORA4Metrics|Not included" msgstr "" +msgid "DORA4Metrics|Number of deployments" +msgstr "" + msgid "DORA4Metrics|Number of incidents divided by the number of deployments to a production environment in the given time period." msgstr "" @@ -16161,9 +16164,6 @@ msgstr "" msgid "DORA4Metrics|Percentage of failed deployments" msgstr "" -msgid "DORA4Metrics|Predicted number of deployments" -msgstr "" - msgid "DORA4Metrics|Show forecast" msgstr "" diff --git a/spec/lib/gitlab/background_migration/backfill_work_item_parent_links_namespace_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_work_item_parent_links_namespace_id_spec.rb new file mode 100644 index 00000000000..aefdc660c85 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_work_item_parent_links_namespace_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemParentLinksNamespaceId, + feature_category: :team_planning, + schema: 20240419035504 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :work_item_parent_links } + let(:backfill_column) { :namespace_id } + let(:backfill_via_table) { :issues } + let(:backfill_via_column) { :namespace_id } + let(:backfill_via_foreign_key) { :work_item_id } + end +end diff --git a/spec/lib/gitlab/database/sharding_key_spec.rb b/spec/lib/gitlab/database/sharding_key_spec.rb index 405ff169431..6f5fd0a465e 100644 --- a/spec/lib/gitlab/database/sharding_key_spec.rb +++ b/spec/lib/gitlab/database/sharding_key_spec.rb @@ -44,8 +44,7 @@ RSpec.describe 'new tables missing sharding_key', feature_category: :cell do 'sprints.group_id', 'subscription_add_on_purchases.namespace_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/444338 'temp_notes_backup.project_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/443667' - *['todos.project_id', 'todos.group_id'], - *['vulnerability_exports.project_id', 'vulnerability_exports.group_id'] + *['todos.project_id', 'todos.group_id'] ] end diff --git a/spec/lib/gitlab/gitaly_client/operation_service_spec.rb b/spec/lib/gitlab/gitaly_client/operation_service_spec.rb index 1ac4efcf3d2..5fc5302737a 100644 --- a/spec/lib/gitlab/gitaly_client/operation_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/operation_service_spec.rb @@ -294,32 +294,41 @@ RSpec.describe Gitlab::GitalyClient::OperationService, feature_category: :source ) {} end - it 'sends a user_merge_branch message', :freeze_time do - first_request = - Gitaly::UserMergeBranchRequest.new( - repository: repository.gitaly_repository, - user: gitaly_user, - commit_id: source_sha, - branch: target_branch, - expected_old_oid: target_sha, - message: message, - timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) - ) - - second_request = Gitaly::UserMergeBranchRequest.new(apply: true) - - expect_next_instance_of(Gitlab::GitalyClient::QueueEnumerator) do |instance| - expect(instance).to receive(:push).with(first_request).and_call_original - expect(instance).to receive(:push).with(second_request).and_call_original - expect(instance).to receive(:close) - end - + it 'succeeds' do expect(subject).to be_a(Gitlab::Git::OperationService::BranchUpdate) expect(subject.newrev).to be_present expect(subject.repo_created).to be(false) expect(subject.branch_created).to be(false) end + it 'receives a bad status' do + expect(client).to receive(:gitaly_client_call) + .and_wrap_original { |original, *args, **kwargs| + response_enum = original.call(*args, **kwargs) + Enumerator.new do |y| + y << response_enum.next + y << response_enum.next + raise 'bad status' + end + } + + expect { subject }.to raise_error(RuntimeError, 'bad status') + end + + it 'receives an unexpected response' do + expect(client).to receive(:gitaly_client_call) + .and_wrap_original { |original, *args, **kwargs| + response_enum = original.call(*args, **kwargs) + Enumerator.new do |y| + y << response_enum.next + y << response_enum.next + y << 'unexpected response' + end + } + + expect { subject }.to raise_error(RuntimeError, 'expected response stream to finish') + end + context 'with an exception with the UserMergeBranchError' do let(:permission_error) do new_detailed_error( @@ -834,17 +843,51 @@ RSpec.describe Gitlab::GitalyClient::OperationService, feature_category: :source end describe '#rebase' do - let(:response) { Gitaly::UserRebaseConfirmableResponse.new } - subject do client.rebase( user, '', - branch: 'master', - branch_sha: 'b83d6e391c22777fca1ed3012fce84f633d7fed0', + branch: 'feature', + branch_sha: '0b4bc9a49b562e85de7cc9e834518ea6828729b9', remote_repository: repository, remote_branch: 'master' - ) + ) {} + end + + context 'with clean repository' do + let(:project) { create(:project, :repository) } + + it 'succeeds' do + expect(subject).to be_present + end + + it 'receives a bad status' do + expect(client).to receive(:gitaly_client_call) + .and_wrap_original { |original, *args, **kwargs| + response_enum = original.call(*args, **kwargs) + Enumerator.new do |y| + y << response_enum.next + y << response_enum.next + raise 'bad status' + end + } + + expect { subject }.to raise_error(RuntimeError, 'bad status') + end + + it 'receives an unexpected response' do + expect(client).to receive(:gitaly_client_call) + .and_wrap_original { |original, *args, **kwargs| + response_enum = original.call(*args, **kwargs) + Enumerator.new do |y| + y << response_enum.next + y << response_enum.next + y << 'unexpected response' + end + } + + expect { subject }.to raise_error(RuntimeError, 'expected response stream to finish') + end end shared_examples '#rebase with an error' do diff --git a/spec/migrations/20240419035508_queue_backfill_work_item_parent_links_namespace_id_spec.rb b/spec/migrations/20240419035508_queue_backfill_work_item_parent_links_namespace_id_spec.rb new file mode 100644 index 00000000000..6d7b60da396 --- /dev/null +++ b/spec/migrations/20240419035508_queue_backfill_work_item_parent_links_namespace_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillWorkItemParentLinksNamespaceId, feature_category: :team_planning do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :work_item_parent_links, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :namespace_id, + :issues, + :namespace_id, + :work_item_id + ] + ) + } + end + end +end