From bc0c5df2f15523e8c4a57682b149526f0cf55b9c Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 29 Jun 2021 18:07:04 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .../merge_request_templates/Documentation.md | 2 +- .../commit/pipelines/pipelines_table.vue | 2 +- app/helpers/application_settings_helper.rb | 2 + app/models/application_setting.rb | 6 + .../application_setting_implementation.rb | 2 + app/models/project.rb | 47 +++- .../merge_request_widget_entity.rb | 2 +- .../application_settings/_mailgun.html.haml | 25 ++ .../application_settings/general.html.haml | 1 + ...ge_request_merge_checks_settings.html.haml | 2 +- config/application.rb | 1 + .../development/mailgun_events_receiver.yml | 8 + ...mailgun_settings_to_application_setting.rb | 10 + db/schema_migrations/20210616185947 | 1 + db/structure.sql | 3 + .../runbooks/planned_failover_multi_node.md | 22 +- .../runbooks/planned_failover_single_node.md | 22 +- doc/administration/lfs/index.md | 49 +++- doc/administration/raketasks/check.md | 4 +- .../reference_architectures/10k_users.md | 4 +- .../reference_architectures/25k_users.md | 2 +- .../reference_architectures/2k_users.md | 2 +- .../reference_architectures/3k_users.md | 2 +- .../reference_architectures/50k_users.md | 184 +++++++++++++- .../reference_architectures/5k_users.md | 2 +- .../reference_architectures/index.md | 1 + doc/api/merge_requests.md | 6 +- doc/api/merge_trains.md | 2 +- doc/api/settings.md | 2 + doc/ci/environments/index.md | 4 +- doc/ci/index.md | 2 +- doc/ci/jobs/job_control.md | 12 +- doc/ci/merge_request_pipelines/index.md | 215 +--------------- .../pipelines_for_merged_results/index.md | 135 +--------- .../merge_trains/index.md | 236 +----------------- doc/ci/parent_child_pipelines.md | 2 +- .../img/merge_train_cancel_v12_0.png | Bin .../img/merge_train_failure.png | Bin .../img/merge_train_immediate_merge_v12_6.png | Bin .../img/merge_train_position_v12_0.png | Bin .../img/merge_train_start_v12_0.png | Bin ...ain_start_when_pipeline_succeeds_v12_0.png | Bin .../img/merged_result_pipeline.png | Bin .../img/pipeline-fork_v13_7.png | Bin doc/ci/pipelines/index.md | 10 +- doc/ci/pipelines/merge_request_pipelines.md | 215 ++++++++++++++++ doc/ci/pipelines/merge_trains.md | 236 ++++++++++++++++++ .../pipelines/pipelines_for_merged_results.md | 135 ++++++++++ doc/ci/review_apps/index.md | 2 +- doc/ci/troubleshooting.md | 20 +- doc/ci/variables/index.md | 2 +- doc/ci/variables/predefined_variables.md | 6 +- doc/ci/yaml/index.md | 12 +- doc/development/cicd/index.md | 4 +- doc/development/code_review.md | 4 +- .../contributing/merge_request_workflow.md | 4 +- doc/development/documentation/index.md | 2 +- doc/development/documentation/structure.md | 122 +++++++-- .../documentation/styleguide/index.md | 7 +- doc/integration/datadog.md | 32 +++ doc/subscriptions/bronze_starter.md | 2 +- doc/topics/git/lfs/index.md | 43 ---- doc/user/application_security/index.md | 2 +- doc/user/project/integrations/overview.md | 2 +- .../project/merge_requests/code_quality.md | 2 +- doc/user/project/merge_requests/drafts.md | 4 +- .../merge_requests/fail_fast_testing.md | 4 +- .../testing_and_reports_in_merge_requests.md | 2 +- doc/user/project/quick_actions.md | 2 +- doc/user/todos.md | 2 +- lib/api/settings.rb | 4 + locale/gitlab.pot | 15 ++ spec/features/admin/admin_settings_spec.rb | 25 ++ spec/frontend/vue_mr_widget/mock_data.js | 4 +- .../100_patch_omniauth_saml_spec.rb | 5 +- spec/models/application_setting_spec.rb | 13 + spec/models/project_spec.rb | 47 ++++ spec/requests/api/settings_spec.rb | 13 + spec/support/omniauth_strategy.rb | 10 +- .../project_ci_cd_settings_shared_examples.rb | 54 ++++ 80 files changed, 1336 insertions(+), 756 deletions(-) create mode 100644 app/views/admin/application_settings/_mailgun.html.haml create mode 100644 config/feature_flags/development/mailgun_events_receiver.yml create mode 100644 db/migrate/20210616185947_add_mailgun_settings_to_application_setting.rb create mode 100644 db/schema_migrations/20210616185947 rename doc/ci/{merge_request_pipelines/pipelines_for_merged_results/merge_trains => pipelines}/img/merge_train_cancel_v12_0.png (100%) rename doc/ci/{merge_request_pipelines/pipelines_for_merged_results/merge_trains => pipelines}/img/merge_train_failure.png (100%) rename doc/ci/{merge_request_pipelines/pipelines_for_merged_results/merge_trains => pipelines}/img/merge_train_immediate_merge_v12_6.png (100%) rename doc/ci/{merge_request_pipelines/pipelines_for_merged_results/merge_trains => pipelines}/img/merge_train_position_v12_0.png (100%) rename doc/ci/{merge_request_pipelines/pipelines_for_merged_results/merge_trains => pipelines}/img/merge_train_start_v12_0.png (100%) rename doc/ci/{merge_request_pipelines/pipelines_for_merged_results/merge_trains => pipelines}/img/merge_train_start_when_pipeline_succeeds_v12_0.png (100%) rename doc/ci/{merge_request_pipelines/pipelines_for_merged_results => pipelines}/img/merged_result_pipeline.png (100%) rename doc/ci/{merge_request_pipelines => pipelines}/img/pipeline-fork_v13_7.png (100%) create mode 100644 doc/ci/pipelines/merge_request_pipelines.md create mode 100644 doc/ci/pipelines/merge_trains.md create mode 100644 doc/ci/pipelines/pipelines_for_merged_results.md create mode 100644 doc/integration/datadog.md create mode 100644 spec/support/shared_examples/models/project_ci_cd_settings_shared_examples.rb diff --git a/.gitlab/merge_request_templates/Documentation.md b/.gitlab/merge_request_templates/Documentation.md index b7c21ed3b67..99ad233c7e0 100644 --- a/.gitlab/merge_request_templates/Documentation.md +++ b/.gitlab/merge_request_templates/Documentation.md @@ -17,7 +17,7 @@ - [ ] Follow the [Documentation Guidelines](https://docs.gitlab.com/ee/development/documentation/) and [Style Guide](https://docs.gitlab.com/ee/development/documentation/styleguide/). - [ ] Ensure that the [product tier badge](https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#product-tier-badges) is added to doc's `h1`. -- [ ] [Request a review](https://docs.gitlab.com/ee/development/code_review.html#dogfooding-the-reviewers-feature) based on the documentation page's metadata and [associated Technical Writer](https://about.gitlab.com/handbook/product/categories/#devops-stages). +- [ ] [Request a review](https://docs.gitlab.com/ee/development/code_review.html#dogfooding-the-reviewers-feature) based on the documentation page's metadata and [associated Technical Writer](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments). To avoid having this MR be added to code verification QA issues, don't add these labels: ~"feature", ~"frontend", ~"backend", ~"bug", or ~"database" diff --git a/app/assets/javascripts/commit/pipelines/pipelines_table.vue b/app/assets/javascripts/commit/pipelines/pipelines_table.vue index ddca5bc7d4f..7aa8b40aef4 100644 --- a/app/assets/javascripts/commit/pipelines/pipelines_table.vue +++ b/app/assets/javascripts/commit/pipelines/pipelines_table.vue @@ -251,7 +251,7 @@ export default { }}

{{ s__('Pipelines|More Information') }} diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 50550fc787d..55ff0d97075 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -343,6 +343,8 @@ module ApplicationSettingsHelper :commit_email_hostname, :protected_ci_variables, :local_markdown_version, + :mailgun_signing_key, + :mailgun_events_enabled, :snowplow_collector_hostname, :snowplow_cookie_domain, :snowplow_enabled, diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 3804f745520..cbde2db803a 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -172,6 +172,11 @@ class ApplicationSetting < ApplicationRecord addressable_url: { enforce_sanitization: true }, if: :gitpod_enabled + validates :mailgun_signing_key, + presence: true, + length: { maximum: 255 }, + if: :mailgun_events_enabled + validates :snowplow_collector_hostname, presence: true, hostname: true, @@ -552,6 +557,7 @@ class ApplicationSetting < ApplicationRecord attr_encrypted :secret_detection_token_revocation_token, encryption_options_base_32_aes_256_gcm attr_encrypted :cloud_license_auth_token, encryption_options_base_32_aes_256_gcm attr_encrypted :external_pipeline_validation_service_token, encryption_options_base_32_aes_256_gcm + attr_encrypted :mailgun_signing_key, encryption_options_base_32_aes_256_gcm.merge(encode: false) validates :disable_feed_token, inclusion: { in: [true, false], message: _('must be a boolean value') } diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index a92ae710959..5b88d7c4a43 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -104,6 +104,8 @@ module ApplicationSettingImplementation issues_create_limit: 300, local_markdown_version: 0, login_recaptcha_protection_enabled: false, + mailgun_signing_key: nil, + mailgun_events_enabled: false, max_artifacts_size: Settings.artifacts['max_size'], max_attachment_size: Settings.gitlab['max_attachment_size'], max_import_size: 0, diff --git a/app/models/project.rb b/app/models/project.rb index 35db99e38db..2ee94273c25 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -424,17 +424,16 @@ class Project < ApplicationRecord delegate :members, to: :team, prefix: true delegate :add_user, :add_users, to: :team delegate :add_guest, :add_reporter, :add_developer, :add_maintainer, :add_role, to: :team - delegate :group_runners_enabled, :group_runners_enabled=, :group_runners_enabled?, to: :ci_cd_settings + delegate :group_runners_enabled, :group_runners_enabled=, to: :ci_cd_settings, allow_nil: true delegate :root_ancestor, to: :namespace, allow_nil: true delegate :last_pipeline, to: :commit, allow_nil: true delegate :external_dashboard_url, to: :metrics_setting, allow_nil: true, prefix: true delegate :dashboard_timezone, to: :metrics_setting, allow_nil: true, prefix: true delegate :default_git_depth, :default_git_depth=, to: :ci_cd_settings, prefix: :ci, allow_nil: true - delegate :forward_deployment_enabled, :forward_deployment_enabled=, :forward_deployment_enabled?, to: :ci_cd_settings, prefix: :ci, allow_nil: true - delegate :job_token_scope_enabled, :job_token_scope_enabled=, :job_token_scope_enabled?, to: :ci_cd_settings, prefix: :ci - delegate :keep_latest_artifact, :keep_latest_artifact=, :keep_latest_artifact?, :keep_latest_artifacts_available?, to: :ci_cd_settings, allow_nil: true - delegate :restrict_user_defined_variables, :restrict_user_defined_variables=, :restrict_user_defined_variables?, - to: :ci_cd_settings, allow_nil: true + delegate :forward_deployment_enabled, :forward_deployment_enabled=, to: :ci_cd_settings, prefix: :ci, allow_nil: true + delegate :job_token_scope_enabled, :job_token_scope_enabled=, to: :ci_cd_settings, prefix: :ci, allow_nil: true + delegate :keep_latest_artifact, :keep_latest_artifact=, to: :ci_cd_settings, allow_nil: true + delegate :restrict_user_defined_variables, :restrict_user_defined_variables=, to: :ci_cd_settings, allow_nil: true delegate :actual_limits, :actual_plan_name, to: :namespace, allow_nil: true delegate :allow_merge_on_skipped_pipeline, :allow_merge_on_skipped_pipeline?, :allow_merge_on_skipped_pipeline=, :has_confluence?, :allow_editing_commit_messages?, @@ -2643,6 +2642,42 @@ class Project < ApplicationRecord end alias_method :container_registry_enabled?, :container_registry_enabled + def ci_forward_deployment_enabled? + return false unless ci_cd_settings + + ci_cd_settings.forward_deployment_enabled? + end + + def ci_job_token_scope_enabled? + return false unless ci_cd_settings + + ci_cd_settings.job_token_scope_enabled? + end + + def restrict_user_defined_variables? + return false unless ci_cd_settings + + ci_cd_settings.restrict_user_defined_variables? + end + + def keep_latest_artifacts_available? + return false unless ci_cd_settings + + ci_cd_settings.keep_latest_artifacts_available? + end + + def keep_latest_artifact? + return false unless ci_cd_settings + + ci_cd_settings.keep_latest_artifact? + end + + def group_runners_enabled? + return false unless ci_cd_settings + + ci_cd_settings.group_runners_enabled? + end + private def set_container_registry_access_level diff --git a/app/serializers/merge_request_widget_entity.rb b/app/serializers/merge_request_widget_entity.rb index 0616d94a1ed..c3e8b66fe12 100644 --- a/app/serializers/merge_request_widget_entity.rb +++ b/app/serializers/merge_request_widget_entity.rb @@ -56,7 +56,7 @@ class MergeRequestWidgetEntity < Grape::Entity end expose :merge_request_pipelines_docs_path do |merge_request| - help_page_path('ci/merge_request_pipelines/index.md') + help_page_path('ci/pipelines/merge_request_pipelines.md') end expose :ci_environments_status_path do |merge_request| diff --git a/app/views/admin/application_settings/_mailgun.html.haml b/app/views/admin/application_settings/_mailgun.html.haml new file mode 100644 index 00000000000..6204f7df5dc --- /dev/null +++ b/app/views/admin/application_settings/_mailgun.html.haml @@ -0,0 +1,25 @@ +- return unless Feature.enabled?(:mailgun_events_receiver) + +- expanded = integration_expanded?('mailgun_') +%section.settings.as-mailgun.no-animate#js-mailgun-settings{ class: ('expanded' if expanded) } + .settings-header + %h4 + = _('Mailgun') + %button.btn.gl-button.btn-default.js-settings-toggle{ type: 'button' } + = expanded ? _('Collapse') : _('Expand') + %p + = _('Configure the %{link} integration.').html_safe % { link: link_to(_('Mailgun events'), 'https://documentation.mailgun.com/en/latest/user_manual.html#webhooks', target: '_blank') } + .settings-content + = form_for @application_setting, url: general_admin_application_settings_path(anchor: 'js-mailgun-settings'), html: { class: 'fieldset-form', id: 'mailgun-settings' } do |f| + = form_errors(@application_setting) if expanded + + %fieldset + .form-group + .form-check + = f.check_box :mailgun_events_enabled, class: 'form-check-input' + = f.label :mailgun_events_enabled, _('Enable Mailgun event receiver'), class: 'form-check-label' + .form-group + = f.label :mailgun_signing_key, _('Mailgun HTTP webhook signing key'), class: 'label-light' + = f.text_field :mailgun_signing_key, class: 'form-control gl-form-input' + + = f.submit _('Save changes'), class: 'gl-button btn btn-confirm' diff --git a/app/views/admin/application_settings/general.html.haml b/app/views/admin/application_settings/general.html.haml index 0fbbef02613..53bdbcd7137 100644 --- a/app/views/admin/application_settings/general.html.haml +++ b/app/views/admin/application_settings/general.html.haml @@ -107,6 +107,7 @@ = render_if_exists 'admin/application_settings/maintenance_mode_settings_form' = render 'admin/application_settings/gitpod' = render 'admin/application_settings/kroki' += render 'admin/application_settings/mailgun' = render 'admin/application_settings/plantuml' = render 'admin/application_settings/sourcegraph' = render_if_exists 'admin/application_settings/slack' diff --git a/app/views/projects/_merge_request_merge_checks_settings.html.haml b/app/views/projects/_merge_request_merge_checks_settings.html.haml index 37598f5760b..fbc58283cbf 100644 --- a/app/views/projects/_merge_request_merge_checks_settings.html.haml +++ b/app/views/projects/_merge_request_merge_checks_settings.html.haml @@ -8,7 +8,7 @@ = form.label :only_allow_merge_if_pipeline_succeeds, class: 'form-check-label' do = s_('ProjectSettings|Pipelines must succeed') .text-secondary - - configuring_pipelines_for_merge_requests_help_link_url = help_page_path('ci/merge_request_pipelines/index.md', anchor: 'configure-pipelines-for-merge-requests') + - configuring_pipelines_for_merge_requests_help_link_url = help_page_path('ci/pipelines/merge_request_pipelines.md', anchor: 'configure-pipelines-for-merge-requests') - configuring_pipelines_for_merge_requests_help_link_start = ''.html_safe % { url: configuring_pipelines_for_merge_requests_help_link_url } = s_('ProjectSettings|To enable this feature, configure pipelines. %{link_start}How to configure pipelines for merge requests?%{link_end}').html_safe % { link_start: configuring_pipelines_for_merge_requests_help_link_start, link_end: ''.html_safe } .form-check.mb-2 diff --git a/config/application.rb b/config/application.rb index 6da5c872cfa..6526be15cd4 100644 --- a/config/application.rb +++ b/config/application.rb @@ -151,6 +151,7 @@ module Gitlab elasticsearch_password search jwt + mailgun_signing_key otp_attempt sentry_dsn trace diff --git a/config/feature_flags/development/mailgun_events_receiver.yml b/config/feature_flags/development/mailgun_events_receiver.yml new file mode 100644 index 00000000000..119d8d34f21 --- /dev/null +++ b/config/feature_flags/development/mailgun_events_receiver.yml @@ -0,0 +1,8 @@ +--- +name: mailgun_events_receiver +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64249 +rollout_issue_url: +milestone: '14.1' +type: development +group: group::expansion +default_enabled: false diff --git a/db/migrate/20210616185947_add_mailgun_settings_to_application_setting.rb b/db/migrate/20210616185947_add_mailgun_settings_to_application_setting.rb new file mode 100644 index 00000000000..8447ff79d12 --- /dev/null +++ b/db/migrate/20210616185947_add_mailgun_settings_to_application_setting.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class AddMailgunSettingsToApplicationSetting < ActiveRecord::Migration[6.1] + def change + add_column :application_settings, :encrypted_mailgun_signing_key, :binary + add_column :application_settings, :encrypted_mailgun_signing_key_iv, :binary + + add_column :application_settings, :mailgun_events_enabled, :boolean, default: false, null: false + end +end diff --git a/db/schema_migrations/20210616185947 b/db/schema_migrations/20210616185947 new file mode 100644 index 00000000000..30275f102dc --- /dev/null +++ b/db/schema_migrations/20210616185947 @@ -0,0 +1 @@ +8d73f4b4b716176afe5a9b0ee3a4ef28bbbc2fe944a18fb66afa8cf8f763e8ac \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 62c559c1c02..56a48be6c4b 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -9522,6 +9522,9 @@ CREATE TABLE application_settings ( diff_max_lines integer DEFAULT 50000 NOT NULL, diff_max_files integer DEFAULT 1000 NOT NULL, valid_runner_registrars character varying[] DEFAULT '{project,group}'::character varying[], + encrypted_mailgun_signing_key bytea, + encrypted_mailgun_signing_key_iv bytea, + mailgun_events_enabled boolean DEFAULT false NOT NULL, CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)), CONSTRAINT app_settings_ext_pipeline_validation_service_url_text_limit CHECK ((char_length(external_pipeline_validation_service_url) <= 255)), CONSTRAINT app_settings_registry_exp_policies_worker_capacity_positive CHECK ((container_registry_expiration_policies_worker_capacity >= 0)), diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md index 4cfe781c7a4..16ae5bde062 100644 --- a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md @@ -19,7 +19,7 @@ This runbook is in **alpha**. For complete, production-ready documentation, see | Geo site | Multi-node | | Secondaries | One | -This runbook will guide you through a planned failover of a multi-node Geo site +This runbook guides you through a planned failover of a multi-node Geo site with one secondary. The following [2000 user reference architecture](../../../../administration/reference_architectures/2k_users.md) is assumed: ```mermaid @@ -46,7 +46,7 @@ graph TD The load balancer node and optional NFS server are omitted for clarity. -This guide will result in the following: +This guide results in the following: 1. An offline primary. 1. A promoted secondary that is now the new primary. @@ -76,7 +76,7 @@ On the **secondary** node: If any objects are failing to replicate, this should be investigated before scheduling the maintenance window. After a planned failover, anything that -failed to replicate will be **lost**. +failed to replicate is **lost**. You can use the [Geo status API](../../../../api/geo_nodes.md#retrieve-project-sync-or-verification-failures-that-occurred-on-the-current-node) @@ -117,10 +117,10 @@ follow these steps to avoid unnecessary data loss: sudo iptables -A INPUT --tcp-dport 443 -j REJECT ``` - From this point, users will be unable to view their data or make changes on the - **primary** node. They will also be unable to log in to the **secondary** node. - However, existing sessions will work for the remainder of the maintenance period, and - public data will be accessible throughout. + From this point, users are unable to view their data or make changes on the + **primary** node. They are also unable to log in to the **secondary** node. + However, existing sessions need to work for the remainder of the maintenance period, and + so public data is accessible throughout. 1. Verify the **primary** node is blocked to HTTP traffic by visiting it in browser via another IP. The server should refuse connection. @@ -170,8 +170,8 @@ follow these steps to avoid unnecessary data loss: 1. [Run an integrity check](../../../raketasks/check.md) to verify the integrity of CI artifacts, LFS objects, and uploads in file storage. - At this point, your **secondary** node will contain an up-to-date copy of everything the - **primary** node has, meaning nothing will be lost when you fail over. + At this point, your **secondary** node contains an up-to-date copy of everything the + **primary** node has, meaning nothing is lost when you fail over. 1. In this final step, you need to permanently disable the **primary** node. @@ -213,7 +213,7 @@ follow these steps to avoid unnecessary data loss: - If you do not have SSH access to the **primary** node, take the machine offline and prevent it from rebooting. Since there are many ways you may prefer to accomplish - this, we will avoid a single recommendation. You may need to: + this, we avoid a single recommendation. You may need to: - Reconfigure the load balancers. - Change DNS records (for example, point the **primary** DNS record to the @@ -248,7 +248,7 @@ issue has been fixed in GitLab 13.4 and later. WARNING: If the secondary node [has been paused](../../../geo/index.md#pausing-and-resuming-replication), this performs a point-in-time recovery to the last known state. -Data that was created on the primary while the secondary was paused will be lost. +Data that was created on the primary while the secondary was paused is lost. 1. SSH in to the PostgreSQL node in the **secondary** and promote PostgreSQL separately: diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md index 6caeddad51a..36c9d46d650 100644 --- a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md @@ -19,7 +19,7 @@ This runbook is in **alpha**. For complete, production-ready documentation, see | Geo site | Single-node | | Secondaries | One | -This runbook will guide you through a planned failover of a single-node Geo site +This runbook guides you through a planned failover of a single-node Geo site with one secondary. The following general architecture is assumed: ```mermaid @@ -34,7 +34,7 @@ graph TD end ``` -This guide will result in the following: +This guide results in the following: 1. An offline primary. 1. A promoted secondary that is now the new primary. @@ -61,7 +61,7 @@ time to complete. If any objects are failing to replicate, this should be investigated before scheduling the maintenance window. After a planned failover, anything that -failed to replicate will be **lost**. +failed to replicate is **lost**. You can use the [Geo status API](../../../../api/geo_nodes.md#retrieve-project-sync-or-verification-failures-that-occurred-on-the-current-node) @@ -102,10 +102,10 @@ follow these steps to avoid unnecessary data loss: sudo iptables -A INPUT --tcp-dport 443 -j REJECT ``` - From this point, users will be unable to view their data or make changes on the - **primary** node. They will also be unable to log in to the **secondary** node. - However, existing sessions will work for the remainder of the maintenance period, and - public data will be accessible throughout. + From this point, users are unable to view their data or make changes on the + **primary** node. They are also unable to log in to the **secondary** node. + However, existing sessions need to work for the remainder of the maintenance period, and + so public data is accessible throughout. 1. Verify the **primary** node is blocked to HTTP traffic by visiting it in browser via another IP. The server should refuse connection. @@ -155,8 +155,8 @@ follow these steps to avoid unnecessary data loss: 1. [Run an integrity check](../../../raketasks/check.md) to verify the integrity of CI artifacts, LFS objects, and uploads in file storage. - At this point, your **secondary** node will contain an up-to-date copy of everything the - **primary** node has, meaning nothing will be lost when you fail over. + At this point, your **secondary** node contains an up-to-date copy of everything the + **primary** node has, meaning nothing is lost when you fail over. 1. In this final step, you need to permanently disable the **primary** node. @@ -198,7 +198,7 @@ follow these steps to avoid unnecessary data loss: - If you do not have SSH access to the **primary** node, take the machine offline and prevent it from rebooting. Since there are many ways you may prefer to accomplish - this, we will avoid a single recommendation. You may need to: + this, we avoid a single recommendation. You may need to: - Reconfigure the load balancers. - Change DNS records (for example, point the **primary** DNS record to the @@ -240,7 +240,7 @@ To promote the secondary node: 1. Run the following command to list out all preflight checks and automatically check if replication and verification are complete before scheduling a planned - failover to ensure the process will go smoothly: + failover to ensure the process goes smoothly: NOTE: In GitLab 13.7 and earlier, if you have a data type with zero items to sync, diff --git a/doc/administration/lfs/index.md b/doc/administration/lfs/index.md index 862c26abac8..793edf8322a 100644 --- a/doc/administration/lfs/index.md +++ b/doc/administration/lfs/index.md @@ -243,7 +243,52 @@ You can see the total storage used for LFS objects on groups and projects: - In the administration area. - In the [groups](../../api/groups.md) and [projects APIs](../../api/projects.md). -## Troubleshooting: `Google::Apis::TransmissionError: execution expired` +## Troubleshooting + +### Missing LFS objects + +An error about a missing LFS object may occur in either of these situations: + +- When migrating LFS objects from disk to object storage, with error messages like: + + ```plaintext + ERROR -- : Failed to transfer LFS object + 006622269c61b41bf14a22bbe0e43be3acf86a4a446afb4250c3794ea47541a7 + with error: No such file or directory @ rb_sysopen - + /var/opt/gitlab/gitlab-rails/shared/lfs-objects/00/66/22269c61b41bf14a22bbe0e43be3acf86a4a446afb4250c3794ea47541a7 + ``` + + (Line breaks have been added for legibility.) + +- When running the + [integrity check for LFS objects](../raketasks/check.md#uploaded-files-integrity) + with the `VERBOSE=1` parameter. + +The database can have records for LFS objects which are not on disk. The database entry may +[prevent a new copy of the object from being pushed](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/49241). +To delete these references: + +1. [Start a rails console](../operations/rails_console.md). +1. Query the object that's reported as missing in the rails console, to return a file path: + + ```ruby + lfs_object = LfsObject.find_by(oid: '006622269c61b41bf14a22bbe0e43be3acf86a4a446afb4250c3794ea47541a7') + lfs_object.file.path + ``` + +1. Check on disk or object storage if it exists: + + ```shell + ls -al /var/opt/gitlab/gitlab-rails/shared/lfs-objects/00/66/22269c61b41bf14a22bbe0e43be3acf86a4a446afb4250c3794ea47541a7 + ``` + +1. If the file is not present, remove the database record via the rails console: + + ```ruby + lfs_object.destroy + ``` + +### `Google::Apis::TransmissionError: execution expired` If LFS integration is configured with Google Cloud Storage and background uploads (`background_upload: true` and `direct_upload: false`), Sidekiq workers may encounter this error. This is because the uploading timed out with very large files. @@ -281,5 +326,5 @@ See more information in [!19581](https://gitlab.com/gitlab-org/gitlab-foss/-/mer - Support for removing unreferenced LFS objects was added in 8.14 onward. - LFS authentications via SSH was added with GitLab 8.12. - Only compatible with the Git LFS client versions 1.1.0 and later, or 1.0.2. -- The storage statistics count each LFS object multiple times for +- The storage statistics count each LFS object for every project linking to it. diff --git a/doc/administration/raketasks/check.md b/doc/administration/raketasks/check.md index f7c91aa6b47..bcc2f838565 100644 --- a/doc/administration/raketasks/check.md +++ b/doc/administration/raketasks/check.md @@ -278,11 +278,11 @@ To delete these references to missing local artifacts (`job.log` files): puts "#{artifact.id} #{artifact.file.path} is missing." ### Allow verification before destroy # artifact.destroy! ### Uncomment to actually destroy end - puts "Count of identified/destroyed invalid references: #{artifacts_deleted}" + puts "Count of identified/destroyed invalid references: #{artifacts_deleted}" ``` ### Delete references to missing LFS objects If `gitlab-rake gitlab:lfs:check VERBOSE=1` detects LFS objects that exist in the database -but not on disk, [follow the procedure in the LFS documentation](../../topics/git/lfs/index.md#missing-lfs-objects) +but not on disk, [follow the procedure in the LFS documentation](../lfs/index.md#missing-lfs-objects) to remove the database entries. diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index f295237357e..55f5c977b02 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -802,7 +802,7 @@ Managed Redis from cloud providers (such as AWS ElastiCache) will work. If these services support high availability, be sure it _isn't_ of the Redis Cluster type. Redis version 5.0 or higher is required, which is included with Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions don't support an -optional count argument to SPOP, which is required for [Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +optional count argument to SPOP, which is required for [Merge Trains](../../ci/pipelines/merge_trains.md). Note the Redis node's IP address or hostname, port, and password (if required). These will be necessary later when configuring the [GitLab application servers](#configure-gitlab-rails). @@ -2535,7 +2535,7 @@ For further information on resource usage, see the [Webservice resources](https: Sidekiq pods should generally have 1 vCPU and 2 GB of memory. [The provided starting point](#cluster-topology) allows the deployment of up to -16 Sidekiq pods. Expand available resources using the 1 vCPU to 2GB memory +14 Sidekiq pods. Expand available resources using the 1 vCPU to 2GB memory ratio for each additional pod. For further information on resource usage, see the [Sidekiq resources](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/#resources). diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index c288196db28..1322f05f93a 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -804,7 +804,7 @@ Managed Redis from cloud providers (such as AWS ElastiCache) will work. If these services support high availability, be sure it _isn't_ of the Redis Cluster type. Redis version 5.0 or higher is required, which is included with Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions don't support an -optional count argument to SPOP, which is required for [Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +optional count argument to SPOP, which is required for [Merge Trains](../../ci/pipelines/merge_trains.md). Note the Redis node's IP address or hostname, port, and password (if required). These will be necessary later when configuring the [GitLab application servers](#configure-gitlab-rails). diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index eae46c32008..ff3db877553 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -324,7 +324,7 @@ to be used with GitLab. Redis version 5.0 or higher is required, as this is what ships with Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions do not support an optional count argument to SPOP which is now required for -[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +[Merge Trains](../../ci/pipelines/merge_trains.md). In addition, GitLab makes use of certain commands like `UNLINK` and `USAGE` which were introduced only in Redis 4. diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 764a11a5cdd..b2d153182e9 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -440,7 +440,7 @@ services support high availability, be sure it is **not** the Redis Cluster type Redis version 5.0 or higher is required, as this is what ships with Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions do not support an optional count argument to SPOP which is now required for -[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +[Merge Trains](../../ci/pipelines/merge_trains.md). Note the Redis node's IP address or hostname, port, and password (if required). These will be necessary when configuring the diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 8cccafcedb9..03c40eea153 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -812,7 +812,7 @@ Managed Redis from cloud providers (such as AWS ElastiCache) will work. If these services support high availability, be sure it _isn't_ of the Redis Cluster type. Redis version 5.0 or higher is required, which is included with Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions don't support an -optional count argument to SPOP, which is required for [Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +optional count argument to SPOP, which is required for [Merge Trains](../../ci/pipelines/merge_trains.md). Note the Redis node's IP address or hostname, port, and password (if required). These will be necessary later when configuring the [GitLab application servers](#configure-gitlab-rails). @@ -2381,6 +2381,188 @@ Read: - The [Gitaly and NFS deprecation notice](../gitaly/index.md#nfs-deprecation-notice). - About the [correct mount options to use](../nfs.md#upgrade-to-gitaly-cluster-or-disable-caching-if-experiencing-data-loss). +## Cloud Native Hybrid reference architecture with Helm Charts (alternative) + +As an alternative approach, you can also run select components of GitLab as Cloud Native +in Kubernetes via our official [Helm Charts](https://docs.gitlab.com/charts/). +In this setup, we support running the equivalent of GitLab Rails and Sidekiq nodes +in a Kubernetes cluster, named Webservice and Sidekiq respectively. In addition, +the following other supporting services are supported: NGINX, Task Runner, Migrations, +Prometheus and Grafana. + +Hybrid installations leverage the benefits of both cloud native and traditional +Kubernetes, you can reap certain cloud native workload management benefits while +the others are deployed in compute VMs with Omnibus as described above in this +page. + +NOTE: +This is an **advanced** setup. Running services in Kubernetes is well known +to be complex. **This setup is only recommended** if you have strong working +knowledge and experience in Kubernetes. The rest of this +section will assume this. + +### Cluster topology + +The following tables and diagram details the hybrid environment using the same formats +as the normal environment above. + +First starting with the components that run in Kubernetes. The recommendations at this +time use Google Cloud’s Kubernetes Engine (GKE) and associated machine types, but the memory +and CPU requirements should translate to most other providers. We hope to update this in the +future with further specific cloud provider details. + +| Service | Nodes(1) | Configuration | GCP | Allocatable CPUs and Memory | +|-------------------------------------------------------|----------|-------------------------|------------------|-----------------------------| +| Webservice | 16 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | 510 vCPU, 472 GB memory | +| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | 15.5 vCPU, 50 GB memory | +| Supporting services such as NGINX, Prometheus, etc. | 2 | 4 vCPU, 15 GB memory | `n1-standard-4` | 7.75 vCPU, 25 GB memory | + + + +1. Nodes configuration is shown as it is forced to ensure pod vcpu / memory ratios and avoid scaling during **performance testing**. + In production deployments there is no need to assign pods to nodes. A minimum of three nodes in three different availability zones is strongly recommended to align with resilient cloud architecture practices. + + +Next are the backend components that run on static compute VMs via Omnibus (or External PaaS +services where applicable): + +| Service | Nodes | Configuration | GCP | +|--------------------------------------------|-------|-------------------------|------------------| +| Consul(1) | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | +| PostgreSQL(1) | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | +| PgBouncer(1) | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | +| Internal load balancing node(3) | 1 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | +| Redis - Cache(2) | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | +| Redis - Queues / Shared State(2) | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | +| Redis Sentinel - Cache(2) | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | +| Redis Sentinel - Queues / Shared State(2) | 3 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | +| Gitaly | 3 | 64 vCPU, 240 GB memory | `n1-standard-64` | +| Praefect | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | +| Praefect PostgreSQL(1) | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | +| Object storage(4) | n/a | n/a | n/a | + + + +1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work, however Azure Database for PostgreSQL is [not recommended](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61) due to performance issues. Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. +4. Should be run on reputable third party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. + + +NOTE: +For all PaaS solutions that involve configuring instances, it is strongly recommended to implement a minimum of three nodes in three different availability zones to align with resilient cloud architecture practices. + +```plantuml +@startuml 50k + +card "Kubernetes via Helm Charts" as kubernetes { + card "**External Load Balancer**" as elb #6a9be7 + + together { + collections "**Webservice** x16" as gitlab #32CD32 + collections "**Sidekiq** x4" as sidekiq #ff8dd1 + } + + card "**Prometheus + Grafana**" as monitor #7FFFD4 + card "**Supporting Services**" as support +} + +card "**Internal Load Balancer**" as ilb #9370DB +collections "**Consul** x3" as consul #e76a9b + +card "Gitaly Cluster" as gitaly_cluster { + collections "**Praefect** x3" as praefect #FF8C00 + collections "**Gitaly** x3" as gitaly #FF8C00 + card "**Praefect PostgreSQL***\n//Non fault-tolerant//" as praefect_postgres #FF8C00 + + praefect -[#FF8C00]-> gitaly + praefect -[#FF8C00]> praefect_postgres +} + +card "Database" as database { + collections "**PGBouncer** x3" as pgbouncer #4EA7FF + card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF + collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF + + pgbouncer -[#4EA7FF]-> postgres_primary + postgres_primary .[#4EA7FF]> postgres_secondary +} + +card "redis" as redis { + collections "**Redis Persistent** x3" as redis_persistent #FF6347 + collections "**Redis Cache** x3" as redis_cache #FF6347 + collections "**Redis Persistent Sentinel** x3" as redis_persistent_sentinel #FF6347 + collections "**Redis Cache Sentinel** x3"as redis_cache_sentinel #FF6347 + + redis_persistent <.[#FF6347]- redis_persistent_sentinel + redis_cache <.[#FF6347]- redis_cache_sentinel +} + +cloud "**Object Storage**" as object_storage #white + +elb -[#6a9be7]-> gitlab +elb -[#6a9be7]-> monitor +elb -[hidden]-> support + +gitlab -[#32CD32]> sidekiq +gitlab -[#32CD32]--> ilb +gitlab -[#32CD32]-> object_storage +gitlab -[#32CD32]---> redis +gitlab -[hidden]--> consul + +sidekiq -[#ff8dd1]--> ilb +sidekiq -[#ff8dd1]-> object_storage +sidekiq -[#ff8dd1]---> redis +sidekiq -[hidden]--> consul + +ilb -[#9370DB]-> gitaly_cluster +ilb -[#9370DB]-> database + +consul .[#e76a9b]-> database +consul .[#e76a9b]-> gitaly_cluster +consul .[#e76a9b,norank]--> redis + +monitor .[#7FFFD4]> consul +monitor .[#7FFFD4]-> database +monitor .[#7FFFD4]-> gitaly_cluster +monitor .[#7FFFD4,norank]--> redis +monitor .[#7FFFD4]> ilb +monitor .[#7FFFD4,norank]u--> elb + +@enduml +``` + +### Resource usage settings + +The following formulas help when calculating how many pods may be deployed within resource constraints. +The [50k reference architecture example values file](https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/examples/ref/50k.yaml) +documents how to apply the calculated configuration to the Helm Chart. + +#### Webservice + +Webservice pods typically need about 1 vCPU and 1.25 GB of memory _per worker_. +Each Webservice pod will consume roughly 4 vCPUs and 5 GB of memory using +the [recommended topology](#cluster-topology) because four worker processes +are created by default and each pod has other small processes running. + +For 50k users we recommend a total Puma worker count of around 320. +With the [provided recommendations](#cluster-topology) this allows the deployment of up to 80 +Webservice pods with 4 workers per pod and 5 pods per node. Expand available resources using +the ratio of 1 vCPU to 1.25 GB of memory _per each worker process_ for each additional +Webservice pod. + +For further information on resource usage, see the [Webservice resources](https://docs.gitlab.com/charts/charts/gitlab/webservice/#resources). + +#### Sidekiq + +Sidekiq pods should generally have 1 vCPU and 2 GB of memory. + +[The provided starting point](#cluster-topology) allows the deployment of up to +14 Sidekiq pods. Expand available resources using the 1 vCPU to 2GB memory +ratio for each additional pod. + +For further information on resource usage, see the [Sidekiq resources](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/#resources). +
Back to setup components diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 96dd66121af..563fad98f59 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -432,7 +432,7 @@ services support high availability, be sure it is **not** the Redis Cluster type Redis version 5.0 or higher is required, as this is what ships with Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions do not support an optional count argument to SPOP which is now required for -[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +[Merge Trains](../../ci/pipelines/merge_trains.md). Note the Redis node's IP address or hostname, port, and password (if required). These will be necessary when configuring the diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md index d3fd845122b..8fee83ec670 100644 --- a/doc/administration/reference_architectures/index.md +++ b/doc/administration/reference_architectures/index.md @@ -72,6 +72,7 @@ The following reference architectures are available: The following Cloud Native Hybrid reference architectures, where select recommended components can be run in Kubernetes, are available: - [Up to 10,000 users](10k_users.md#cloud-native-hybrid-reference-architecture-with-helm-charts-alternative) +- [Up to 50,000 users](50k_users.md#cloud-native-hybrid-reference-architecture-with-helm-charts-alternative) A GitLab [Premium or Ultimate](https://about.gitlab.com/pricing/#self-managed) license is required to get assistance from Support with troubleshooting the [2,000 users](2k_users.md) diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index 1c2e764e8e4..bc8c0397a3a 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -1006,15 +1006,15 @@ Parameters: > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/31722) in GitLab 12.3. -Create a new [pipeline for a merge request](../ci/merge_request_pipelines/index.md). +Create a new [pipeline for a merge request](../ci/pipelines/merge_request_pipelines.md). A pipeline created via this endpoint doesn't run a regular branch/tag pipeline. It requires `.gitlab-ci.yml` to be configured with `only: [merge_requests]` to create jobs. The new pipeline can be: - A detached merge request pipeline. -- A [pipeline for merged results](../ci/merge_request_pipelines/pipelines_for_merged_results/index.md) - if the [project setting is enabled](../ci/merge_request_pipelines/pipelines_for_merged_results/index.md#enable-pipelines-for-merged-results). +- A [pipeline for merged results](../ci/pipelines/pipelines_for_merged_results.md) + if the [project setting is enabled](../ci/pipelines/pipelines_for_merged_results.md#enable-pipelines-for-merged-results). ```plaintext POST /projects/:id/merge_requests/:merge_request_iid/pipelines diff --git a/doc/api/merge_trains.md b/doc/api/merge_trains.md index 079fa006744..2b52166d954 100644 --- a/doc/api/merge_trains.md +++ b/doc/api/merge_trains.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Merge Trains API **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36146) in GitLab 12.9. -> - Using this API you can consume [Merge Train](../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md) entries. +> - Using this API you can consume [Merge Train](../ci/pipelines/merge_trains.md) entries. Every API call to merge trains must be authenticated with Developer or higher [permissions](../user/permissions.md). diff --git a/doc/api/settings.md b/doc/api/settings.md index ce62cd193f7..d49dca96dfd 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -328,6 +328,8 @@ listed in the descriptions of the relevant settings. | `issues_create_limit` | integer | no | Max number of issue creation requests per minute per user. Disabled by default.| | `keep_latest_artifact` | boolean | no | Prevent the deletion of the artifacts from the most recent successful jobs, regardless of the expiry time. Enabled by default. | | `local_markdown_version` | integer | no | Increase this value when any cached Markdown should be invalidated. | +| `mailgun_signing_key` | string | no | The Mailgun HTTP webhook signing key for receiving events from webhook | +| `mailgun_events_enabled` | boolean | no | Enable Mailgun event receiver. | | `maintenance_mode_message` | string | no | **(PREMIUM)** Message displayed when instance is in maintenance mode. | | `maintenance_mode` | boolean | no | **(PREMIUM)** When instance is in maintenance mode, non-administrative users can sign in with read-only access and make read-only API requests. | | `max_artifacts_size` | integer | no | Maximum artifacts size in MB. | diff --git a/doc/ci/environments/index.md b/doc/ci/environments/index.md index 903d5c8e647..e647a704a87 100644 --- a/doc/ci/environments/index.md +++ b/doc/ci/environments/index.md @@ -302,7 +302,7 @@ Note the following: - If the environment URL isn't valid (for example, the URL is malformed), the system doesn't update the environment URL. - If the script that runs in `stop_review` exists only in your repository and therefore can't use - `GIT_STRATEGY: none`, configure [pipelines for merge requests](../../ci/merge_request_pipelines/index.md) + `GIT_STRATEGY: none`, configure [pipelines for merge requests](../../ci/pipelines/merge_request_pipelines.md) for these jobs. This ensures that runners can fetch the repository even after a feature branch is deleted. For more information, see [Ref Specs for Runners](../pipelines/index.md#ref-specs-for-runners). @@ -408,7 +408,7 @@ the `stop_review` job might not be included in all pipelines that include the The job with [`action: stop` might not run](#the-job-with-action-stop-doesnt-run) if it's in a later stage than the job that started the environment. -If you can't use [pipelines for merge requests](../merge_request_pipelines/index.md), +If you can't use [pipelines for merge requests](../pipelines/merge_request_pipelines.md), set the [`GIT_STRATEGY`](../runners/configure_runners.md#git-strategy) to `none` in the `stop_review` job. Then the [runner](https://docs.gitlab.com/runner/) doesn't try to check out the code after the branch is deleted. diff --git a/doc/ci/index.md b/doc/ci/index.md index 1cd78f6d5ef..cbb0a359fbe 100644 --- a/doc/ci/index.md +++ b/doc/ci/index.md @@ -74,7 +74,7 @@ GitLab CI/CD supports numerous configuration options: | [Git submodules for CI/CD](git_submodules.md) | Configure jobs for using Git submodules. | | [SSH keys for CI/CD](ssh_keys/index.md) | Using SSH keys in your CI pipelines. | | [Pipeline triggers](triggers/index.md) | Trigger pipelines through the API. | -| [Pipelines for Merge Requests](merge_request_pipelines/index.md) | Design a pipeline structure for running a pipeline in merge requests. | +| [Pipelines for Merge Requests](pipelines/merge_request_pipelines.md) | Design a pipeline structure for running a pipeline in merge requests. | | [Integrate with Kubernetes clusters](../user/project/clusters/index.md) | Connect your project to Google Kubernetes Engine (GKE) or an existing Kubernetes cluster. | | [Optimize GitLab and GitLab Runner for large repositories](large_repositories/index.md) | Recommended strategies for handling large repositories. | | [`.gitlab-ci.yml` full reference](yaml/index.md) | All the attributes you can use with GitLab CI/CD. | diff --git a/doc/ci/jobs/job_control.md b/doc/ci/jobs/job_control.md index c0c4f256633..03aad57b57d 100644 --- a/doc/ci/jobs/job_control.md +++ b/doc/ci/jobs/job_control.md @@ -47,7 +47,7 @@ job: ``` - If the pipeline is for a merge request, the first rule matches, and the job - is added to the [merge request pipeline](../merge_request_pipelines/index.md) + is added to the [merge request pipeline](../pipelines/merge_request_pipelines.md) with attributes of: - `when: manual` (manual job) - `allow_failure: true` (the pipeline continues running even if the manual job is not run) @@ -220,7 +220,7 @@ check the value of the `$CI_PIPELINE_SOURCE` variable: | `chat` | For pipelines created by using a [GitLab ChatOps](../chatops/index.md) command. | | `external` | When you use CI services other than GitLab. | | `external_pull_request_event` | When an external pull request on GitHub is created or updated. See [Pipelines for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests). | -| `merge_request_event` | For pipelines created when a merge request is created or updated. Required to enable [merge request pipelines](../merge_request_pipelines/index.md), [merged results pipelines](../merge_request_pipelines/pipelines_for_merged_results/index.md), and [merge trains](../merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). | +| `merge_request_event` | For pipelines created when a merge request is created or updated. Required to enable [merge request pipelines](../pipelines/merge_request_pipelines.md), [merged results pipelines](../pipelines/pipelines_for_merged_results.md), and [merge trains](../pipelines/merge_trains.md). | | `parent_pipeline` | For pipelines triggered by a [parent/child pipeline](../parent_child_pipelines.md) with `rules`. Use this pipeline source in the child pipeline configuration so that it can be triggered by the parent pipeline. | | `pipeline` | For [multi-project pipelines](../multi_project_pipelines.md) created by [using the API with `CI_JOB_TOKEN`](../multi_project_pipelines.md#create-multi-project-pipelines-by-using-the-api), or the [`trigger`](../yaml/index.md#trigger) keyword. | | `push` | For pipelines triggered by a `git push` event, including for branches and tags. | @@ -243,7 +243,7 @@ job: - if: '$CI_PIPELINE_SOURCE == "push"' ``` -The following example runs the job as a `when: on_success` job in [merge request pipelines](../merge_request_pipelines/index.md) +The following example runs the job as a `when: on_success` job in [merge request pipelines](../pipelines/merge_request_pipelines.md) and scheduled pipelines. It does not run in any other pipeline type. ```yaml @@ -382,7 +382,7 @@ Read more about how to use `only:changes` and `except:changes`: #### Use `only:changes` with pipelines for merge requests -With [pipelines for merge requests](../merge_request_pipelines/index.md), +With [pipelines for merge requests](../pipelines/merge_request_pipelines.md), it's possible to define a job to be created based on files modified in a merge request. @@ -432,7 +432,7 @@ properly corrects any failures from previous pipelines. #### Use `only:changes` without pipelines for merge requests -Without [pipelines for merge requests](../merge_request_pipelines/index.md), pipelines +Without [pipelines for merge requests](../pipelines/merge_request_pipelines.md), pipelines run on branches or tags that don't have an explicit association with a merge request. In this case, a previous SHA is used to calculate the diff, which is equivalent to `git diff HEAD~`. This can result in some unexpected behavior, including: @@ -511,7 +511,7 @@ types the variables can control for: - Branch pipelines that run for Git `push` events to a branch, like new commits or tags. - Tag pipelines that run only when a new Git tag is pushed to a branch. -- [Merge request pipelines](../merge_request_pipelines/index.md) that run for changes +- [Merge request pipelines](../pipelines/merge_request_pipelines.md) that run for changes to a merge request, like new commits or selecting the **Run pipeline** button in a merge request's pipelines tab. - [Scheduled pipelines](../pipelines/schedules.md). diff --git a/doc/ci/merge_request_pipelines/index.md b/doc/ci/merge_request_pipelines/index.md index e710df4f397..c852800d0a9 100644 --- a/doc/ci/merge_request_pipelines/index.md +++ b/doc/ci/merge_request_pipelines/index.md @@ -1,215 +1,8 @@ --- -stage: Verify -group: Pipeline Execution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: reference, index -last_update: 2019-07-03 +redirect_to: '../pipelines/merge_request_pipelines.md' --- -# Pipelines for merge requests **(FREE)** +This document was moved to [another location](../pipelines/merge_request_pipelines.md). -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/15310) in GitLab 11.6. - -In a [basic configuration](../pipelines/pipeline_architectures.md#basic-pipelines), GitLab runs a pipeline each time -changes are pushed to a branch. - -If you want the pipeline to run jobs **only** on commits associated with a merge request, -you can use *pipelines for merge requests*. - -In the UI, these pipelines are labeled as `detached`. Otherwise, these pipelines are the same -as other pipelines. - -Pipelines for merge requests can run when you: - -- Create a new merge request. -- Commit changes to the source branch for the merge request. -- Select the **Run pipeline** button from the **Pipelines** tab in the merge request. - -If you use this feature with [merge when pipeline succeeds](../../user/project/merge_requests/merge_when_pipeline_succeeds.md), -pipelines for merge requests take precedence over other pipelines. - -## Prerequisites - -To enable pipelines for merge requests: - -- Your repository must be a GitLab repository, not an - [external repository](../ci_cd_for_external_repos/index.md). -- You must have the Developer [role](../../user/permissions.md) - to run a pipeline for merge requests. - -## Configure pipelines for merge requests - -To configure pipelines for merge requests, you must configure your [CI/CD configuration file](../yaml/index.md). -To do this, you can use [`rules`](#use-rules-to-run-pipelines-for-merge-requests) or [`only/except`](#use-only-or-except-to-run-pipelines-for-merge-requests). - -### Use `rules` to run pipelines for merge requests - -GitLab recommends that you use the `rules` keyword, which is available in -[`workflow:rules` templates](../yaml/index.md#workflowrules-templates). - -### Use `only` or `except` to run pipelines for merge requests - -You can use the `only/except` keywords. However, with this method, you must specify `only: - merge_requests` for each job. - -In the following example, the pipeline contains a `test` job that is configured to run on merge requests. -The `build` and `deploy` jobs don't have the `only: - merge_requests` keyword, -so they don't run on merge requests. - -```yaml -build: - stage: build - script: ./build - only: - - main - -test: - stage: test - script: ./test - only: - - merge_requests - -deploy: - stage: deploy - script: ./deploy - only: - - main -``` - -#### Exclude specific jobs - -When you use `only: [merge_requests]`, only jobs with -that keyword are run in the context of a merge request. No other jobs run. - -However, you can invert this behavior and have all of your jobs run except -for one or two. For example, you might have a pipeline with jobs `A`, `B`, and `C`, and you want: - -- All pipelines to always run `A` and `B`. -- `C` to run only for merge requests. - -To achieve this outcome, configure your `.gitlab-ci.yml` file as follows: - -```yaml -.only-default: &only-default - only: - - main - - merge_requests - - tags - -A: - <<: *only-default - script: - - ... - -B: - <<: *only-default - script: - - ... - -C: - script: - - ... - only: - - merge_requests -``` - -- `A` and `B` always run, because they get the `only:` rule to execute in all cases. -- `C` only runs for merge requests. It doesn't run for any pipeline - except a merge request pipeline. - -In this example, you don't have to add the `only:` rule to all of your jobs to make -them always run. You can use this format to set up a Review App, which helps to -save resources. - -#### Exclude specific branches - -Branch refs use this format: `refs/heads/my-feature-branch`. -Merge request refs use this format: `refs/merge-requests/:iid/head`. - -Because of this difference, the following configuration does not work as expected: - -```yaml -# Does not exclude a branch named "docs-my-fix"! -test: - only: [merge_requests] - except: [/^docs-/] -``` - -Instead, use the -[`$CI_COMMIT_REF_NAME` predefined environment -variable](../variables/predefined_variables.md) in -combination with -[`only:variables`](../yaml/index.md#onlyvariables--exceptvariables) to -accomplish this behavior: - -```yaml -test: - only: [merge_requests] - except: - variables: - - $CI_COMMIT_REF_NAME =~ /^docs-/ -``` - -## Run pipelines in the parent project for merge requests from a forked project **(PREMIUM)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217451) in GitLab 13.3. -> - [Moved](https://about.gitlab.com/blog/2021/01/26/new-gitlab-product-subscription-model/) to GitLab Premium in 13.9. - -By default, external contributors who work in forks can't create pipelines in the -parent project. When a merge request that comes from a fork triggers a pipeline: - -- The pipeline is created and runs in the fork (source) project, not the parent (target) project. -- The pipeline uses the fork project's CI/CD configuration and resources. - -If a pipeline runs in a fork, a **fork** badge appears for the pipeline in the merge request. - -![Pipeline ran in fork](img/pipeline-fork_v13_7.png) - -Sometimes parent project members want the pipeline to run in the parent -project. They may want to ensure that the post-merge pipeline passes in the parent project. -For example, a fork project could try to use a corrupted runner that doesn't execute -test scripts properly, but reports a passed pipeline. Reviewers in the parent project -could mistakenly trust the merge request because it passed a faked pipeline. - -Parent project members with at least the [Developer role](../../user/permissions.md) -can create pipelines in the parent project for merge requests -from a forked project. In the merge request, go to the **Pipelines** tab and select -**Run pipeline**. - -WARNING: -Fork merge requests can contain malicious code that tries to steal secrets in the -parent project when the pipeline runs, even before merge. As a reviewer, you must carefully -check the changes in the merge request before triggering the pipeline. GitLab shows -a warning that you must accept before you can trigger the pipeline. - -## Predefined variables available for pipelines for merge requests - -When you use pipelines for merge requests, [additional predefined variables](../variables/predefined_variables.md#predefined-variables-for-merge-request-pipelines) are available to the CI/CD jobs. -These variables contain information from the associated merge request, so that you can -integrate your job with the [GitLab Merge Request API](../../api/merge_requests.md). - -## Troubleshooting - -### Two pipelines created when pushing to a merge request - -If you are experiencing duplicated pipelines when using `rules`, take a look at -the [important differences between `rules` and `only`/`except`](../jobs/job_control.md#avoid-duplicate-pipelines), -which helps you get your starting configuration correct. - -If you are seeing two pipelines when using `only/except`, please see the caveats -related to using `only/except` above (or, consider moving to `rules`). - -In [GitLab 13.7](https://gitlab.com/gitlab-org/gitlab/-/issues/201845) and later, -you can add `workflow:rules` to [switch from branch pipelines to merge request pipelines](../yaml/index.md#switch-between-branch-pipelines-and-merge-request-pipelines). -After a merge request is open on the branch, the pipeline switches to a merge request pipeline. - -### Two pipelines created when pushing an invalid CI configuration file - -Pushing to a branch with an invalid CI configuration file can trigger -the creation of two types of failed pipelines. One pipeline is a failed merge request -pipeline, and the other is a failed branch pipeline, but both are caused by the same -invalid configuration. - -## Related topics - -- [Pipelines for merged results](pipelines_for_merged_results/index.md). -- [Merge trains](pipelines_for_merged_results/merge_trains/index.md). + + diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md index 67b390d5acc..76a79ba1356 100644 --- a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md @@ -1,135 +1,8 @@ --- -stage: Verify -group: Pipeline Execution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: reference -last_update: 2019-07-03 +redirect_to: '../../pipelines/pipelines_for_merged_results.md' --- -# Pipelines for merged results **(PREMIUM)** +This document was moved to [another location](../../pipelines/pipelines_for_merged_results.md). -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7380) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.10. - -When you submit a merge request, you are requesting to merge changes from a -source branch into a target branch. By default, the CI pipeline runs jobs -against the source branch. - -With *pipelines for merged results*, the pipeline runs as if the changes from -the source branch have already been merged into the target branch. The commit shown for the pipeline does not exist on the source or target branches but represents the combined target and source branches. - -![Merge request widget for merged results pipeline](img/merged_result_pipeline.png) - -If the pipeline fails due to a problem in the target branch, you can wait until the -target is fixed and re-run the pipeline. -This new pipeline runs as if the source is merged with the updated target, and you -don't need to rebase. - -The pipeline does not automatically run when the target branch changes. Only changes -to the source branch trigger a new pipeline. If a long time has passed since the last successful -pipeline, you may want to re-run it before merge, to ensure that the source changes -can still be successfully merged into the target. - -When the merge request can't be merged, the pipeline runs against the source branch only. For example, when: - -- The target branch has changes that conflict with the changes in the source branch. -- The merge request is a [**Draft** merge request](../../../user/project/merge_requests/drafts.md). - -In these cases, the pipeline runs as a [pipeline for merge requests](../index.md) -and is labeled as `detached`. If these cases no longer exist, new pipelines -again run against the merged results. - -Any user who has developer [permissions](../../../user/permissions.md) can run a -pipeline for merged results. - -## Prerequisites - -To enable pipelines for merge results: - -- You must have the [Maintainer role](../../../user/permissions.md). -- You must be using [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner) 11.9 or later. -- You must not be using - [fast forward merges](../../../user/project/merge_requests/fast_forward_merge.md) yet. - To follow progress, see [#58226](https://gitlab.com/gitlab-org/gitlab/-/issues/26996). -- Your repository must be a GitLab repository, not an - [external repository](../../ci_cd_for_external_repos/index.md). - -## Enable pipelines for merged results - -To enable pipelines for merged results for your project: - -1. [Configure your CI/CD configuration file](../index.md#configure-pipelines-for-merge-requests) - so that the pipeline or individual jobs run for merge requests. -1. Visit your project's **Settings > General** and expand **Merge requests**. -1. Check **Enable merged results pipelines**. -1. Click **Save changes**. - -WARNING: -If you select the check box but don't configure your CI/CD to use -pipelines for merge requests, your merge requests may become stuck in an -unresolved state or your pipelines may be dropped. - -## Using Merge Trains - -When you enable [Pipelines for merged results](#pipelines-for-merged-results), -GitLab [automatically displays](merge_trains/index.md#add-a-merge-request-to-a-merge-train) -a **Start/Add Merge Train button**. - -Generally, this is a safer option than merging merge requests immediately, because your -merge request is evaluated with an expected post-merge result before the actual -merge happens. - -For more information, read the [documentation on Merge Trains](merge_trains/index.md). - -## Automatic pipeline cancellation - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12996) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. - -GitLab CI/CD can detect the presence of redundant pipelines, and cancels them -to conserve CI resources. - -When a user merges a merge request immediately within an ongoing merge -train, the train is reconstructed, because it recreates the expected -post-merge commit and pipeline. In this case, the merge train may already -have pipelines running against the previous expected post-merge commit. -These pipelines are considered redundant and are automatically -canceled. - -## Troubleshooting - -### Pipelines for merged results not created even with new change pushed to merge request - -Can be caused by some disabled feature flags. Please make sure that -the following feature flags are enabled on your GitLab instance: - -- `:merge_ref_auto_sync` - -To check and set these feature flag values, please ask an administrator to: - -1. Log into the Rails console of the GitLab instance: - - ```shell - sudo gitlab-rails console - ``` - -1. Check if the flags are enabled or not: - - ```ruby - Feature.enabled?(:merge_ref_auto_sync) - ``` - -1. If needed, enable the feature flags: - - ```ruby - Feature.enable(:merge_ref_auto_sync) - ``` - -### Intermittently pipelines fail by `fatal: reference is not a tree:` error - -Since pipelines for merged results are a run on a merge ref of a merge request -(`refs/merge-requests//merge`), the Git reference could be overwritten at an -unexpected timing. For example, when a source or target branch is advanced. -In this case, the pipeline fails because of `fatal: reference is not a tree:` error, -which indicates that the checkout-SHA is not found in the merge ref. - -This behavior was improved at GitLab 12.4 by introducing [Persistent pipeline refs](../../troubleshooting.md#fatal-reference-is-not-a-tree-error). -You should be able to create pipelines at any timings without concerning the error. + + diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md index f2fb0a86982..8b2316fd340 100644 --- a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md @@ -1,236 +1,8 @@ --- -stage: Verify -group: Pipeline Execution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: reference -last_update: 2019-07-03 +redirect_to: '../../../pipelines/merge_trains.md' --- -# Merge Trains **(PREMIUM)** +This document was moved to [another location](../../../pipelines/merge_trains.md). -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9186) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.0. -> - [Squash and merge](../../../../user/project/merge_requests/squash_and_merge.md) support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13001) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.6. - -For more information about why you might want to use Merge Trains, read [How merge trains keep your master green](https://about.gitlab.com/blog/2020/01/30/all-aboard-merge-trains/). - -When [pipelines for merged results](../index.md#pipelines-for-merged-results) are -enabled, the pipeline jobs run as if the changes from your source branch have already -been merged into the target branch. - -However, the target branch may be changing rapidly. When you're ready to merge, -if you haven't run the pipeline in a while, the target branch may have already changed. -Merging now could introduce breaking changes. - -*Merge trains* can prevent this from happening. A merge train is a queued list of merge -requests, each waiting to be merged into the target branch. - -Many merge requests can be added to the train. Each merge request runs its own merged results pipeline, -which includes the changes from all of the other merge requests in *front* of it on the train. -All the pipelines run in parallel, to save time. - -If the pipeline for a merge request fails, the breaking changes are not merged, and the target -branch is unaffected. The merge request is removed from the train, and all pipelines behind it restart. - -If the pipeline for the merge request at the front of the train completes successfully, -the changes are merged into the target branch, and the other pipelines continue to -run. - -To add a merge request to a merge train, you need [permissions](../../../../user/permissions.md) to push to the target branch. - -Each merge train can run a maximum of **twenty** pipelines in parallel. -If more than twenty merge requests are added to the merge train, the merge requests -are queued until a slot in the merge train is free. There is no limit to the -number of merge requests that can be queued. - -## Merge train example - -Three merge requests (`A`, `B` and `C`) are added to a merge train in order, which -creates three merged results pipelines that run in parallel: - -1. The first pipeline runs on the changes from `A` combined with the target branch. -1. The second pipeline runs on the changes from `A` and `B` combined with the target branch. -1. The third pipeline runs on the changes from `A`, `B`, and `C` combined with the target branch. - -If the pipeline for `B` fails, it is removed from the train. The pipeline for -`C` restarts with the `A` and `C` changes, but without the `B` changes. - -If `A` then completes successfully, it merges into the target branch, and `C` continues -to run. If more merge requests are added to the train, they now include the `A` -changes that are included in the target branch, and the `C` changes that are from -the merge request already in the train. - -Read more about [how merge trains keep your master green](https://about.gitlab.com/blog/2020/01/30/all-aboard-merge-trains/). - - -Watch this video for a demonstration on [how parallel execution -of Merge Trains can prevent commits from breaking the default -branch](https://www.youtube.com/watch?v=D4qCqXgZkHQ). - -## Prerequisites - -To enable merge trains: - -- You must have the [Maintainer role](../../../../user/permissions.md). -- You must be using [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner) 11.9 or later. -- In GitLab 13.0 and later, you need [Redis](https://redis.io/) 5.0 or later. -- Your repository must be a GitLab repository, not an - [external repository](../../../ci_cd_for_external_repos/index.md). - -## Enable merge trains - -To enable merge trains for your project: - -1. If you are on a self-managed GitLab instance, ensure the [feature flag](#merge-trains-feature-flag) is set correctly. -1. [Configure your CI/CD configuration file](../../index.md#configure-pipelines-for-merge-requests) - so that the pipeline or individual jobs run for merge requests. -1. Visit your project's **Settings > General** and expand **Merge requests**. -1. In the **Merge method** section, verify that **Merge commit** is selected. - You cannot use **Merge commit with semi-linear history** or **Fast-forward merge** with merge trains. -1. In the **Merge options** section, select **Enable merged results pipelines.** (if not already selected) and **Enable merge trains.** -1. Click **Save changes** - -In GitLab 13.5 and earlier, there is only one checkbox, named -**Enable merge trains and pipelines for merged results**. - -WARNING: -If you select the check box but don't configure your CI/CD to use -pipelines for merge requests, your merge requests may become stuck in an -unresolved state or your pipelines may be dropped. - -## Start a merge train - -To start a merge train: - -1. Visit a merge request. -1. Click the **Start merge train** button. - -![Start merge train](img/merge_train_start_v12_0.png) - -Other merge requests can now be added to the train. - -## Add a merge request to a merge train - -To add a merge request to a merge train: - -1. Visit a merge request. -1. Click the **Add to merge train** button. - -If pipelines are already running for the merge request, you cannot add the merge request -to the train. Instead, you can schedule to add the merge request to a merge train **when the latest -pipeline succeeds**. - -![Add to merge train when pipeline succeeds](img/merge_train_start_when_pipeline_succeeds_v12_0.png) - -## Remove a merge request from a merge train - -1. Visit a merge request. -1. Click the **Remove from merge train** button. - -![Cancel merge train](img/merge_train_cancel_v12_0.png) - -If you want to add the merge request to a merge train again later, you can. - -## View a merge request's current position on the merge train - -After a merge request has been added to the merge train, the merge request's -current position is displayed under the pipeline widget: - -![Merge train position indicator](img/merge_train_position_v12_0.png) - -## Immediately merge a merge request with a merge train - -If you have a high-priority merge request (for example, a critical patch) that must -be merged urgently, you can bypass the merge train by using the **Merge Immediately** option. -This is the fastest option to get the change merged into the target branch. - -![Merge Immediately](img/merge_train_immediate_merge_v12_6.png) - -WARNING: -Each time you merge a merge request immediately, the current merge train -is recreated and all pipelines restart. - -## Troubleshooting - -### Merge request dropped from the merge train immediately - -If a merge request is not mergeable (for example, it's a draft merge request, there is a merge -conflict, etc.), your merge request is dropped from the merge train automatically. - -In these cases, the reason for dropping the merge request is in the **system notes**. - -To check the reason: - -1. Open the merge request that was dropped from the merge train. -1. Open the **Discussion** tab. -1. Find a system note that includes either: - - The text **... removed this merge request from the merge train because ...** - - **... aborted this merge request from the merge train because ...** - The reason is given in the text after the **because ...** phrase. - -![Merge Train Failure](img/merge_train_failure.png) - -### Merge When Pipeline Succeeds cannot be chosen - -[Merge When Pipeline Succeeds](../../../../user/project/merge_requests/merge_when_pipeline_succeeds.md) -is currently unavailable when Merge Trains are enabled. - -See [the related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/12267) -for more information. - -### Merge Train Pipeline cannot be retried - -When a pipeline for merge trains fails the merge request is dropped from the train and the pipeline can't be retried. -Pipelines for merge trains run on the merged result of the changes in the merge request and -the changes from other merge requests already on the train. If the merge request is dropped from the train, -the merged result is out of date and the pipeline can't be retried. - -Instead, you should [add the merge request to the train](#add-a-merge-request-to-a-merge-train) -again, which triggers a new pipeline. - -### Unable to add to merge train with message "The pipeline for this merge request failed." - -Sometimes the **Start/Add to Merge Train** button is not available and the merge request says, -"The pipeline for this merge request failed. Please retry the job or push a new commit to fix the failure." - -This issue occurs when [**Pipelines must succeed**](../../../../user/project/merge_requests/merge_when_pipeline_succeeds.md#only-allow-merge-requests-to-be-merged-if-the-pipeline-succeeds) -is enabled in **Settings > General > Merge requests**. This option requires that you -run a new successful pipeline before you can re-add a merge request to a merge train. - -Merge trains ensure that each pipeline has succeeded before a merge happens, so -you can clear the **Pipelines must succeed** check box and keep -**Enable merge trains and pipelines for merged results** (merge trains) enabled. - -If you want to keep the **Pipelines must succeed** option enabled along with Merge -Trains, create a new pipeline for merged results when this error occurs: - -1. Go to the **Pipelines** tab and click **Run pipeline**. -1. Click **Start/Add to merge train when pipeline succeeds**. - -See [the related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/35135) -for more information. - -### Merge Trains feature flag **(PREMIUM SELF)** - -In [GitLab 13.6 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/244831), -you can [enable or disable merge trains in the project settings](#enable-merge-trains). - -In GitLab 13.5 and earlier, merge trains are automatically enabled when -[pipelines for merged results](../index.md#pipelines-for-merged-results) are enabled. -To use pipelines for merged results without using merge trains, you can enable a -[feature flag](../../../../user/feature_flags.md) that blocks the merge trains feature. - -[GitLab administrators with access to the GitLab Rails console](../../../../administration/feature_flags.md) -can enable the feature flag to disable merge trains: - -```ruby -Feature.enable(:disable_merge_trains) -``` - -After you enable this feature flag, all existing merge trains are cancelled and -the **Start/Add to Merge Train** button no longer appears in merge requests. - -To disable the feature flag, and enable merge trains again: - -```ruby -Feature.disable(:disable_merge_trains) -``` + + diff --git a/doc/ci/parent_child_pipelines.md b/doc/ci/parent_child_pipelines.md index 05adcd42898..ee4e7e65b0e 100644 --- a/doc/ci/parent_child_pipelines.md +++ b/doc/ci/parent_child_pipelines.md @@ -99,7 +99,7 @@ microservice_a: ## Merge Request child pipelines -To trigger a child pipeline as a [Merge Request Pipeline](merge_request_pipelines/index.md) we need to: +To trigger a child pipeline as a [Merge Request Pipeline](pipelines/merge_request_pipelines.md) we need to: - Set the trigger job to run on merge requests: diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_cancel_v12_0.png b/doc/ci/pipelines/img/merge_train_cancel_v12_0.png similarity index 100% rename from doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_cancel_v12_0.png rename to doc/ci/pipelines/img/merge_train_cancel_v12_0.png diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_failure.png b/doc/ci/pipelines/img/merge_train_failure.png similarity index 100% rename from doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_failure.png rename to doc/ci/pipelines/img/merge_train_failure.png diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_immediate_merge_v12_6.png b/doc/ci/pipelines/img/merge_train_immediate_merge_v12_6.png similarity index 100% rename from doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_immediate_merge_v12_6.png rename to doc/ci/pipelines/img/merge_train_immediate_merge_v12_6.png diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_position_v12_0.png b/doc/ci/pipelines/img/merge_train_position_v12_0.png similarity index 100% rename from doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_position_v12_0.png rename to doc/ci/pipelines/img/merge_train_position_v12_0.png diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_start_v12_0.png b/doc/ci/pipelines/img/merge_train_start_v12_0.png similarity index 100% rename from doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_start_v12_0.png rename to doc/ci/pipelines/img/merge_train_start_v12_0.png diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_start_when_pipeline_succeeds_v12_0.png b/doc/ci/pipelines/img/merge_train_start_when_pipeline_succeeds_v12_0.png similarity index 100% rename from doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_start_when_pipeline_succeeds_v12_0.png rename to doc/ci/pipelines/img/merge_train_start_when_pipeline_succeeds_v12_0.png diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/img/merged_result_pipeline.png b/doc/ci/pipelines/img/merged_result_pipeline.png similarity index 100% rename from doc/ci/merge_request_pipelines/pipelines_for_merged_results/img/merged_result_pipeline.png rename to doc/ci/pipelines/img/merged_result_pipeline.png diff --git a/doc/ci/merge_request_pipelines/img/pipeline-fork_v13_7.png b/doc/ci/pipelines/img/pipeline-fork_v13_7.png similarity index 100% rename from doc/ci/merge_request_pipelines/img/pipeline-fork_v13_7.png rename to doc/ci/pipelines/img/pipeline-fork_v13_7.png diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index 5041b56bba7..621ad4a60d9 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -54,12 +54,12 @@ Pipelines can be configured in many different ways: - [Parent-Child pipelines](../parent_child_pipelines.md) break down complex pipelines into one parent pipeline that can trigger multiple child sub-pipelines, which all run in the same project and with the same SHA. -- [Pipelines for Merge Requests](../merge_request_pipelines/index.md) run for merge +- [Pipelines for Merge Requests](../pipelines/merge_request_pipelines.md) run for merge requests only (rather than for every commit). -- [Pipelines for Merged Results](../merge_request_pipelines/pipelines_for_merged_results/index.md) +- [Pipelines for Merged Results](../pipelines/pipelines_for_merged_results.md) are merge request pipelines that act as though the changes from the source branch have already been merged into the target branch. -- [Merge Trains](../merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md) +- [Merge Trains](../pipelines/merge_trains.md) use pipelines for merged results to queue merges one after the other. ## Configure a pipeline @@ -89,13 +89,13 @@ This table lists the refspecs injected for each pipeline type: |--------------- |---------------------------------------- | | Pipeline for Branches | `+:refs/pipelines/` and `+refs/heads/:refs/remotes/origin/` | | pipeline for Tags | `+:refs/pipelines/` and `+refs/tags/:refs/tags/` | -| [Pipeline for Merge Requests](../merge_request_pipelines/index.md) | `+:refs/pipelines/` | +| [Pipeline for Merge Requests](../pipelines/merge_request_pipelines.md) | `+:refs/pipelines/` | The refs `refs/heads/` and `refs/tags/` exist in your project repository. GitLab generates the special ref `refs/pipelines/` during a running pipeline job. This ref can be created even after the associated branch or tag has been deleted. It's therefore useful in some features such as [automatically stopping an environment](../environments/index.md#stopping-an-environment), -and [merge trains](../merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md) +and [merge trains](../pipelines/merge_trains.md) that might run pipelines after branch deletion. ### View pipelines diff --git a/doc/ci/pipelines/merge_request_pipelines.md b/doc/ci/pipelines/merge_request_pipelines.md new file mode 100644 index 00000000000..29c12551f12 --- /dev/null +++ b/doc/ci/pipelines/merge_request_pipelines.md @@ -0,0 +1,215 @@ +--- +stage: Verify +group: Pipeline Execution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +type: reference, index +last_update: 2019-07-03 +--- + +# Pipelines for merge requests **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/15310) in GitLab 11.6. + +In a [basic configuration](pipeline_architectures.md#basic-pipelines), GitLab runs a pipeline each time +changes are pushed to a branch. + +If you want the pipeline to run jobs **only** on commits associated with a merge request, +you can use *pipelines for merge requests*. + +In the UI, these pipelines are labeled as `detached`. Otherwise, these pipelines are the same +as other pipelines. + +Pipelines for merge requests can run when you: + +- Create a new merge request. +- Commit changes to the source branch for the merge request. +- Select the **Run pipeline** button from the **Pipelines** tab in the merge request. + +If you use this feature with [merge when pipeline succeeds](../../user/project/merge_requests/merge_when_pipeline_succeeds.md), +pipelines for merge requests take precedence over other pipelines. + +## Prerequisites + +To enable pipelines for merge requests: + +- Your repository must be a GitLab repository, not an + [external repository](../ci_cd_for_external_repos/index.md). +- You must have the Developer [role](../../user/permissions.md) + to run a pipeline for merge requests. + +## Configure pipelines for merge requests + +To configure pipelines for merge requests, you must configure your [CI/CD configuration file](../yaml/index.md). +To do this, you can use [`rules`](#use-rules-to-run-pipelines-for-merge-requests) or [`only/except`](#use-only-or-except-to-run-pipelines-for-merge-requests). + +### Use `rules` to run pipelines for merge requests + +GitLab recommends that you use the `rules` keyword, which is available in +[`workflow:rules` templates](../yaml/index.md#workflowrules-templates). + +### Use `only` or `except` to run pipelines for merge requests + +You can use the `only/except` keywords. However, with this method, you must specify `only: - merge_requests` for each job. + +In the following example, the pipeline contains a `test` job that is configured to run on merge requests. +The `build` and `deploy` jobs don't have the `only: - merge_requests` keyword, +so they don't run on merge requests. + +```yaml +build: + stage: build + script: ./build + only: + - main + +test: + stage: test + script: ./test + only: + - merge_requests + +deploy: + stage: deploy + script: ./deploy + only: + - main +``` + +#### Exclude specific jobs + +When you use `only: [merge_requests]`, only jobs with +that keyword are run in the context of a merge request. No other jobs run. + +However, you can invert this behavior and have all of your jobs run except +for one or two. For example, you might have a pipeline with jobs `A`, `B`, and `C`, and you want: + +- All pipelines to always run `A` and `B`. +- `C` to run only for merge requests. + +To achieve this outcome, configure your `.gitlab-ci.yml` file as follows: + +```yaml +.only-default: &only-default + only: + - main + - merge_requests + - tags + +A: + <<: *only-default + script: + - ... + +B: + <<: *only-default + script: + - ... + +C: + script: + - ... + only: + - merge_requests +``` + +- `A` and `B` always run, because they get the `only:` rule to execute in all cases. +- `C` only runs for merge requests. It doesn't run for any pipeline + except a merge request pipeline. + +In this example, you don't have to add the `only:` rule to all of your jobs to make +them always run. You can use this format to set up a Review App, which helps to +save resources. + +#### Exclude specific branches + +Branch refs use this format: `refs/heads/my-feature-branch`. +Merge request refs use this format: `refs/merge-requests/:iid/head`. + +Because of this difference, the following configuration does not work as expected: + +```yaml +# Does not exclude a branch named "docs-my-fix"! +test: + only: [merge_requests] + except: [/^docs-/] +``` + +Instead, use the +[`$CI_COMMIT_REF_NAME` predefined environment +variable](../variables/predefined_variables.md) in +combination with +[`only:variables`](../yaml/index.md#onlyvariables--exceptvariables) to +accomplish this behavior: + +```yaml +test: + only: [merge_requests] + except: + variables: + - $CI_COMMIT_REF_NAME =~ /^docs-/ +``` + +## Run pipelines in the parent project for merge requests from a forked project **(PREMIUM)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217451) in GitLab 13.3. +> - [Moved](https://about.gitlab.com/blog/2021/01/26/new-gitlab-product-subscription-model/) to GitLab Premium in 13.9. + +By default, external contributors who work in forks can't create pipelines in the +parent project. When a merge request that comes from a fork triggers a pipeline: + +- The pipeline is created and runs in the fork (source) project, not the parent (target) project. +- The pipeline uses the fork project's CI/CD configuration and resources. + +If a pipeline runs in a fork, a **fork** badge appears for the pipeline in the merge request. + +![Pipeline ran in fork](img/pipeline-fork_v13_7.png) + +Sometimes parent project members want the pipeline to run in the parent +project. They may want to ensure that the post-merge pipeline passes in the parent project. +For example, a fork project could try to use a corrupted runner that doesn't execute +test scripts properly, but reports a passed pipeline. Reviewers in the parent project +could mistakenly trust the merge request because it passed a faked pipeline. + +Parent project members with at least the [Developer role](../../user/permissions.md) +can create pipelines in the parent project for merge requests +from a forked project. In the merge request, go to the **Pipelines** tab and select +**Run pipeline**. + +WARNING: +Fork merge requests can contain malicious code that tries to steal secrets in the +parent project when the pipeline runs, even before merge. As a reviewer, you must carefully +check the changes in the merge request before triggering the pipeline. GitLab shows +a warning that you must accept before you can trigger the pipeline. + +## Predefined variables available for pipelines for merge requests + +When you use pipelines for merge requests, [additional predefined variables](../variables/predefined_variables.md#predefined-variables-for-merge-request-pipelines) are available to the CI/CD jobs. +These variables contain information from the associated merge request, so that you can +integrate your job with the [GitLab Merge Request API](../../api/merge_requests.md). + +## Troubleshooting + +### Two pipelines created when pushing to a merge request + +If you are experiencing duplicated pipelines when using `rules`, take a look at +the [important differences between `rules` and `only`/`except`](../jobs/job_control.md#avoid-duplicate-pipelines), +which helps you get your starting configuration correct. + +If you are seeing two pipelines when using `only/except`, please see the caveats +related to using `only/except` above (or, consider moving to `rules`). + +In [GitLab 13.7](https://gitlab.com/gitlab-org/gitlab/-/issues/201845) and later, +you can add `workflow:rules` to [switch from branch pipelines to merge request pipelines](../yaml/index.md#switch-between-branch-pipelines-and-merge-request-pipelines). +After a merge request is open on the branch, the pipeline switches to a merge request pipeline. + +### Two pipelines created when pushing an invalid CI configuration file + +Pushing to a branch with an invalid CI configuration file can trigger +the creation of two types of failed pipelines. One pipeline is a failed merge request +pipeline, and the other is a failed branch pipeline, but both are caused by the same +invalid configuration. + +## Related topics + +- [Pipelines for merged results](pipelines_for_merged_results.md). +- [Merge trains](merge_trains.md). diff --git a/doc/ci/pipelines/merge_trains.md b/doc/ci/pipelines/merge_trains.md new file mode 100644 index 00000000000..3e6ad071d7e --- /dev/null +++ b/doc/ci/pipelines/merge_trains.md @@ -0,0 +1,236 @@ +--- +stage: Verify +group: Pipeline Execution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +type: reference +last_update: 2019-07-03 +--- + +# Merge Trains **(PREMIUM)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9186) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.0. +> - [Squash and merge](../../user/project/merge_requests/squash_and_merge.md) support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13001) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.6. + +For more information about why you might want to use Merge Trains, read [How merge trains keep your master green](https://about.gitlab.com/blog/2020/01/30/all-aboard-merge-trains/). + +When [pipelines for merged results](pipelines_for_merged_results.md) are +enabled, the pipeline jobs run as if the changes from your source branch have already +been merged into the target branch. + +However, the target branch may be changing rapidly. When you're ready to merge, +if you haven't run the pipeline in a while, the target branch may have already changed. +Merging now could introduce breaking changes. + +*Merge trains* can prevent this from happening. A merge train is a queued list of merge +requests, each waiting to be merged into the target branch. + +Many merge requests can be added to the train. Each merge request runs its own merged results pipeline, +which includes the changes from all of the other merge requests in *front* of it on the train. +All the pipelines run in parallel, to save time. + +If the pipeline for a merge request fails, the breaking changes are not merged, and the target +branch is unaffected. The merge request is removed from the train, and all pipelines behind it restart. + +If the pipeline for the merge request at the front of the train completes successfully, +the changes are merged into the target branch, and the other pipelines continue to +run. + +To add a merge request to a merge train, you need [permissions](../../user/permissions.md) to push to the target branch. + +Each merge train can run a maximum of **twenty** pipelines in parallel. +If more than twenty merge requests are added to the merge train, the merge requests +are queued until a slot in the merge train is free. There is no limit to the +number of merge requests that can be queued. + +## Merge train example + +Three merge requests (`A`, `B` and `C`) are added to a merge train in order, which +creates three merged results pipelines that run in parallel: + +1. The first pipeline runs on the changes from `A` combined with the target branch. +1. The second pipeline runs on the changes from `A` and `B` combined with the target branch. +1. The third pipeline runs on the changes from `A`, `B`, and `C` combined with the target branch. + +If the pipeline for `B` fails, it is removed from the train. The pipeline for +`C` restarts with the `A` and `C` changes, but without the `B` changes. + +If `A` then completes successfully, it merges into the target branch, and `C` continues +to run. If more merge requests are added to the train, they now include the `A` +changes that are included in the target branch, and the `C` changes that are from +the merge request already in the train. + +Read more about [how merge trains keep your master green](https://about.gitlab.com/blog/2020/01/30/all-aboard-merge-trains/). + + +Watch this video for a demonstration on [how parallel execution +of Merge Trains can prevent commits from breaking the default +branch](https://www.youtube.com/watch?v=D4qCqXgZkHQ). + +## Prerequisites + +To enable merge trains: + +- You must have the [Maintainer role](../../user/permissions.md). +- You must be using [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner) 11.9 or later. +- In GitLab 13.0 and later, you need [Redis](https://redis.io/) 5.0 or later. +- Your repository must be a GitLab repository, not an + [external repository](../ci_cd_for_external_repos/index.md). + +## Enable merge trains + +To enable merge trains for your project: + +1. If you are on a self-managed GitLab instance, ensure the [feature flag](#merge-trains-feature-flag) is set correctly. +1. [Configure your CI/CD configuration file](merge_request_pipelines.md#configure-pipelines-for-merge-requests) + so that the pipeline or individual jobs run for merge requests. +1. Visit your project's **Settings > General** and expand **Merge requests**. +1. In the **Merge method** section, verify that **Merge commit** is selected. + You cannot use **Merge commit with semi-linear history** or **Fast-forward merge** with merge trains. +1. In the **Merge options** section, select **Enable merged results pipelines.** (if not already selected) and **Enable merge trains.** +1. Click **Save changes** + +In GitLab 13.5 and earlier, there is only one checkbox, named +**Enable merge trains and pipelines for merged results**. + +WARNING: +If you select the check box but don't configure your CI/CD to use +pipelines for merge requests, your merge requests may become stuck in an +unresolved state or your pipelines may be dropped. + +## Start a merge train + +To start a merge train: + +1. Visit a merge request. +1. Click the **Start merge train** button. + +![Start merge train](img/merge_train_start_v12_0.png) + +Other merge requests can now be added to the train. + +## Add a merge request to a merge train + +To add a merge request to a merge train: + +1. Visit a merge request. +1. Click the **Add to merge train** button. + +If pipelines are already running for the merge request, you cannot add the merge request +to the train. Instead, you can schedule to add the merge request to a merge train **when the latest +pipeline succeeds**. + +![Add to merge train when pipeline succeeds](img/merge_train_start_when_pipeline_succeeds_v12_0.png) + +## Remove a merge request from a merge train + +1. Visit a merge request. +1. Click the **Remove from merge train** button. + +![Cancel merge train](img/merge_train_cancel_v12_0.png) + +If you want to add the merge request to a merge train again later, you can. + +## View a merge request's current position on the merge train + +After a merge request has been added to the merge train, the merge request's +current position is displayed under the pipeline widget: + +![Merge train position indicator](img/merge_train_position_v12_0.png) + +## Immediately merge a merge request with a merge train + +If you have a high-priority merge request (for example, a critical patch) that must +be merged urgently, you can bypass the merge train by using the **Merge Immediately** option. +This is the fastest option to get the change merged into the target branch. + +![Merge Immediately](img/merge_train_immediate_merge_v12_6.png) + +WARNING: +Each time you merge a merge request immediately, the current merge train +is recreated and all pipelines restart. + +## Troubleshooting + +### Merge request dropped from the merge train immediately + +If a merge request is not mergeable (for example, it's a draft merge request, there is a merge +conflict, etc.), your merge request is dropped from the merge train automatically. + +In these cases, the reason for dropping the merge request is in the **system notes**. + +To check the reason: + +1. Open the merge request that was dropped from the merge train. +1. Open the **Discussion** tab. +1. Find a system note that includes either: + - The text **... removed this merge request from the merge train because ...** + - **... aborted this merge request from the merge train because ...** + The reason is given in the text after the **because ...** phrase. + +![Merge Train Failure](img/merge_train_failure.png) + +### Merge When Pipeline Succeeds cannot be chosen + +[Merge When Pipeline Succeeds](../../user/project/merge_requests/merge_when_pipeline_succeeds.md) +is currently unavailable when Merge Trains are enabled. + +See [the related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/12267) +for more information. + +### Merge Train Pipeline cannot be retried + +When a pipeline for merge trains fails the merge request is dropped from the train and the pipeline can't be retried. +Pipelines for merge trains run on the merged result of the changes in the merge request and +the changes from other merge requests already on the train. If the merge request is dropped from the train, +the merged result is out of date and the pipeline can't be retried. + +Instead, you should [add the merge request to the train](#add-a-merge-request-to-a-merge-train) +again, which triggers a new pipeline. + +### Unable to add to merge train with message "The pipeline for this merge request failed." + +Sometimes the **Start/Add to Merge Train** button is not available and the merge request says, +"The pipeline for this merge request failed. Please retry the job or push a new commit to fix the failure." + +This issue occurs when [**Pipelines must succeed**](../../user/project/merge_requests/merge_when_pipeline_succeeds.md#only-allow-merge-requests-to-be-merged-if-the-pipeline-succeeds) +is enabled in **Settings > General > Merge requests**. This option requires that you +run a new successful pipeline before you can re-add a merge request to a merge train. + +Merge trains ensure that each pipeline has succeeded before a merge happens, so +you can clear the **Pipelines must succeed** check box and keep +**Enable merge trains and pipelines for merged results** (merge trains) enabled. + +If you want to keep the **Pipelines must succeed** option enabled along with Merge +Trains, create a new pipeline for merged results when this error occurs: + +1. Go to the **Pipelines** tab and click **Run pipeline**. +1. Click **Start/Add to merge train when pipeline succeeds**. + +See [the related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/35135) +for more information. + +### Merge Trains feature flag **(PREMIUM SELF)** + +In [GitLab 13.6 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/244831), +you can [enable or disable merge trains in the project settings](#enable-merge-trains). + +In GitLab 13.5 and earlier, merge trains are automatically enabled when +[pipelines for merged results](pipelines_for_merged_results.md) are enabled. +To use pipelines for merged results without using merge trains, you can enable a +[feature flag](../../user/feature_flags.md) that blocks the merge trains feature. + +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can enable the feature flag to disable merge trains: + +```ruby +Feature.enable(:disable_merge_trains) +``` + +After you enable this feature flag, all existing merge trains are cancelled and +the **Start/Add to Merge Train** button no longer appears in merge requests. + +To disable the feature flag, and enable merge trains again: + +```ruby +Feature.disable(:disable_merge_trains) +``` diff --git a/doc/ci/pipelines/pipelines_for_merged_results.md b/doc/ci/pipelines/pipelines_for_merged_results.md new file mode 100644 index 00000000000..910c6415cfd --- /dev/null +++ b/doc/ci/pipelines/pipelines_for_merged_results.md @@ -0,0 +1,135 @@ +--- +stage: Verify +group: Pipeline Execution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +type: reference +last_update: 2019-07-03 +--- + +# Pipelines for merged results **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7380) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.10. + +When you submit a merge request, you are requesting to merge changes from a +source branch into a target branch. By default, the CI pipeline runs jobs +against the source branch. + +With *pipelines for merged results*, the pipeline runs as if the changes from +the source branch have already been merged into the target branch. The commit shown for the pipeline does not exist on the source or target branches but represents the combined target and source branches. + +![Merge request widget for merged results pipeline](img/merged_result_pipeline.png) + +If the pipeline fails due to a problem in the target branch, you can wait until the +target is fixed and re-run the pipeline. +This new pipeline runs as if the source is merged with the updated target, and you +don't need to rebase. + +The pipeline does not automatically run when the target branch changes. Only changes +to the source branch trigger a new pipeline. If a long time has passed since the last successful +pipeline, you may want to re-run it before merge, to ensure that the source changes +can still be successfully merged into the target. + +When the merge request can't be merged, the pipeline runs against the source branch only. For example, when: + +- The target branch has changes that conflict with the changes in the source branch. +- The merge request is a [**Draft** merge request](../../user/project/merge_requests/drafts.md). + +In these cases, the pipeline runs as a [pipeline for merge requests](merge_request_pipelines.md) +and is labeled as `detached`. If these cases no longer exist, new pipelines +again run against the merged results. + +Any user who has developer [permissions](../../user/permissions.md) can run a +pipeline for merged results. + +## Prerequisites + +To enable pipelines for merge results: + +- You must have the [Maintainer role](../../user/permissions.md). +- You must be using [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner) 11.9 or later. +- You must not be using + [fast forward merges](../../user/project/merge_requests/fast_forward_merge.md) yet. + To follow progress, see [#58226](https://gitlab.com/gitlab-org/gitlab/-/issues/26996). +- Your repository must be a GitLab repository, not an + [external repository](../ci_cd_for_external_repos/index.md). + +## Enable pipelines for merged results + +To enable pipelines for merged results for your project: + +1. [Configure your CI/CD configuration file](merge_request_pipelines.md#configure-pipelines-for-merge-requests) + so that the pipeline or individual jobs run for merge requests. +1. Visit your project's **Settings > General** and expand **Merge requests**. +1. Check **Enable merged results pipelines**. +1. Click **Save changes**. + +WARNING: +If you select the check box but don't configure your CI/CD to use +pipelines for merge requests, your merge requests may become stuck in an +unresolved state or your pipelines may be dropped. + +## Using Merge Trains + +When you enable [Pipelines for merged results](#pipelines-for-merged-results), +GitLab [automatically displays](merge_trains.md#add-a-merge-request-to-a-merge-train) +a **Start/Add Merge Train button**. + +Generally, this is a safer option than merging merge requests immediately, because your +merge request is evaluated with an expected post-merge result before the actual +merge happens. + +For more information, read the [documentation on Merge Trains](merge_trains.md). + +## Automatic pipeline cancellation + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12996) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3. + +GitLab CI/CD can detect the presence of redundant pipelines, and cancels them +to conserve CI resources. + +When a user merges a merge request immediately within an ongoing merge +train, the train is reconstructed, because it recreates the expected +post-merge commit and pipeline. In this case, the merge train may already +have pipelines running against the previous expected post-merge commit. +These pipelines are considered redundant and are automatically +canceled. + +## Troubleshooting + +### Pipelines for merged results not created even with new change pushed to merge request + +Can be caused by some disabled feature flags. Please make sure that +the following feature flags are enabled on your GitLab instance: + +- `:merge_ref_auto_sync` + +To check and set these feature flag values, please ask an administrator to: + +1. Log into the Rails console of the GitLab instance: + + ```shell + sudo gitlab-rails console + ``` + +1. Check if the flags are enabled or not: + + ```ruby + Feature.enabled?(:merge_ref_auto_sync) + ``` + +1. If needed, enable the feature flags: + + ```ruby + Feature.enable(:merge_ref_auto_sync) + ``` + +### Intermittently pipelines fail by `fatal: reference is not a tree:` error + +Since pipelines for merged results are a run on a merge ref of a merge request +(`refs/merge-requests//merge`), the Git reference could be overwritten at an +unexpected timing. For example, when a source or target branch is advanced. +In this case, the pipeline fails because of `fatal: reference is not a tree:` error, +which indicates that the checkout-SHA is not found in the merge ref. + +This behavior was improved at GitLab 12.4 by introducing [Persistent pipeline refs](../troubleshooting.md#fatal-reference-is-not-a-tree-error). +You should be able to create pipelines at any timings without concerning the error. diff --git a/doc/ci/review_apps/index.md b/doc/ci/review_apps/index.md index 8f10ad9151b..4956ecb8af6 100644 --- a/doc/ci/review_apps/index.md +++ b/doc/ci/review_apps/index.md @@ -254,7 +254,7 @@ to replace those values at runtime when each review app is created: variable. - `data-merge-request-id` is the merge request ID, which can be found by the `CI_MERGE_REQUEST_IID` variable. `CI_MERGE_REQUEST_IID` is available only if - [`only: [merge_requests]`](../merge_request_pipelines/index.md) + [`only: [merge_requests]`](../pipelines/merge_request_pipelines.md) is used and the merge request is created. - `data-mr-url` is the URL of the GitLab instance and is the same for all review apps. diff --git a/doc/ci/troubleshooting.md b/doc/ci/troubleshooting.md index 310118b26aa..fc4a222f87a 100644 --- a/doc/ci/troubleshooting.md +++ b/doc/ci/troubleshooting.md @@ -66,11 +66,11 @@ if you are using that type: and run separate pipelines in the same project. You can also [dynamically generate the child pipeline's configuration](parent_child_pipelines.md#dynamic-child-pipelines) at runtime. -- [Pipelines for Merge Requests](merge_request_pipelines/index.md): Run a pipeline +- [Pipelines for Merge Requests](pipelines/merge_request_pipelines.md): Run a pipeline in the context of a merge request. - - [Pipelines for Merge Results](merge_request_pipelines/pipelines_for_merged_results/index.md): + - [Pipelines for Merge Results](pipelines/pipelines_for_merged_results.md): Pipelines for merge requests that run on the combined source and target branch - - [Merge Trains](merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md): + - [Merge Trains](pipelines/merge_trains.md): Multiple pipelines for merged results that queue and run automatically before changes are merged. @@ -80,7 +80,7 @@ There are troubleshooting guides available for some CI/CD features and related t - [Container Registry](../user/packages/container_registry/index.md#troubleshooting-the-gitlab-container-registry) - [GitLab Runner](https://docs.gitlab.com/runner/faq/) -- [Merge Trains](merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md#troubleshooting) +- [Merge Trains](pipelines/merge_trains.md#troubleshooting) - [Docker Build](docker/using_docker_build.md#troubleshooting) - [Environments](environments/deployment_safety.md#ensure-only-one-deployment-job-runs-at-a-time) @@ -170,8 +170,8 @@ a branch to its remote repository. To illustrate the problem, suppose you've had This is because the previous pipeline cannot find a checkout-SHA (which is associated with the pipeline record) from the `example` branch that the commit history has already been overwritten by the force-push. -Similarly, [Pipelines for merged results](merge_request_pipelines/pipelines_for_merged_results/index.md) -might have failed intermittently due to [the same reason](merge_request_pipelines/pipelines_for_merged_results/index.md#intermittently-pipelines-fail-by-fatal-reference-is-not-a-tree-error). +Similarly, [Pipelines for merged results](pipelines/pipelines_for_merged_results.md) +might have failed intermittently due to [the same reason](pipelines/pipelines_for_merged_results.md#intermittently-pipelines-fail-by-fatal-reference-is-not-a-tree-error). As of GitLab 12.4, we've improved this behavior by persisting pipeline refs exclusively. To illustrate its life cycle: @@ -211,7 +211,7 @@ is displayed. If the pipeline is still running, the **Merge** button is replaced with the **Merge when pipeline succeeds** button. -If [**Merge Trains**](merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md) +If [**Merge Trains**](pipelines/merge_trains.md) are enabled, the button is either **Add to merge train** or **Add to merge train when pipeline succeeds**. **(PREMIUM)** #### "A CI/CD pipeline must run and be successful before merge" message @@ -224,9 +224,9 @@ should disable **Pipelines must succeed** so you can accept merge requests. ### "The pipeline for this merge request did not complete. Push a new commit to fix the failure or check the troubleshooting documentation to see other possible actions." message -This message is shown if the [merge request pipeline](merge_request_pipelines/index.md), -[merged results pipeline](merge_request_pipelines/pipelines_for_merged_results/index.md), -or [merge train pipeline](merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md) +This message is shown if the [merge request pipeline](pipelines/merge_request_pipelines.md), +[merged results pipeline](pipelines/pipelines_for_merged_results.md), +or [merge train pipeline](pipelines/merge_trains.md) has failed or been canceled. If a merge request pipeline or merged result pipeline was canceled or failed, you can: diff --git a/doc/ci/variables/index.md b/doc/ci/variables/index.md index d7c9e3e164d..a7f971bc4b5 100644 --- a/doc/ci/variables/index.md +++ b/doc/ci/variables/index.md @@ -328,7 +328,7 @@ runs on a [protected branch](../../user/project/protected_branches.md) or Review all merge requests that introduce changes to the `.gitlab-ci.yml` file before you: -- [Run a pipeline in the parent project for a merge request submitted from a forked project](../merge_request_pipelines/index.md#run-pipelines-in-the-parent-project-for-merge-requests-from-a-forked-project). +- [Run a pipeline in the parent project for a merge request submitted from a forked project](../pipelines/merge_request_pipelines.md#run-pipelines-in-the-parent-project-for-merge-requests-from-a-forked-project). - Merge the changes. The following example shows malicious code in a `.gitlab-ci.yml` file: diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index 0180da79c2c..b6dd9446644 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -125,7 +125,7 @@ There are also [Kubernetes-specific deployment variables](../../user/project/clu These variables are available when: -- The pipelines [are merge request pipelines](../merge_request_pipelines/index.md). +- The pipelines [are merge request pipelines](../pipelines/merge_request_pipelines.md). - The merge request is open. | Variable | GitLab | Runner | Description | @@ -141,12 +141,12 @@ These variables are available when: | `CI_MERGE_REQUEST_PROJECT_URL` | 11.6 | all | The URL of the project of the merge request. For example, `http://192.168.10.15:3000/namespace/awesome-project`. | | `CI_MERGE_REQUEST_REF_PATH` | 11.6 | all | The ref path of the merge request. For example, `refs/merge-requests/1/head`. | | `CI_MERGE_REQUEST_SOURCE_BRANCH_NAME` | 11.6 | all | The source branch name of the merge request. | -| `CI_MERGE_REQUEST_SOURCE_BRANCH_SHA` | 11.9 | all | The HEAD SHA of the source branch of the merge request. The variable is empty in merge request pipelines. The SHA is present only in [merged results pipelines](../merge_request_pipelines/pipelines_for_merged_results/index.md). **(PREMIUM)** | +| `CI_MERGE_REQUEST_SOURCE_BRANCH_SHA` | 11.9 | all | The HEAD SHA of the source branch of the merge request. The variable is empty in merge request pipelines. The SHA is present only in [merged results pipelines](../pipelines/pipelines_for_merged_results.md). **(PREMIUM)** | | `CI_MERGE_REQUEST_SOURCE_PROJECT_ID` | 11.6 | all | The ID of the source project of the merge request. | | `CI_MERGE_REQUEST_SOURCE_PROJECT_PATH` | 11.6 | all | The path of the source project of the merge request. | | `CI_MERGE_REQUEST_SOURCE_PROJECT_URL` | 11.6 | all | The URL of the source project of the merge request. | | `CI_MERGE_REQUEST_TARGET_BRANCH_NAME` | 11.6 | all | The target branch name of the merge request. | -| `CI_MERGE_REQUEST_TARGET_BRANCH_SHA` | 11.9 | all | The HEAD SHA of the target branch of the merge request. The variable is empty in merge request pipelines. The SHA is present only in [merged results pipelines](../merge_request_pipelines/pipelines_for_merged_results/index.md). **(PREMIUM)** | +| `CI_MERGE_REQUEST_TARGET_BRANCH_SHA` | 11.9 | all | The HEAD SHA of the target branch of the merge request. The variable is empty in merge request pipelines. The SHA is present only in [merged results pipelines](../pipelines/pipelines_for_merged_results.md). **(PREMIUM)** | | `CI_MERGE_REQUEST_TITLE` | 11.9 | all | The title of the merge request. | | `CI_MERGE_REQUEST_EVENT_TYPE` | 12.3 | all | The event type of the merge request. Can be `detached`, `merged_result` or `merge_train`. | | `CI_MERGE_REQUEST_DIFF_ID` | 13.7 | all | The version of the merge request diff. | diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index 5e1f44abf0e..e2322d640e4 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -298,9 +298,9 @@ makes your pipelines run for branches and tags. Branch pipeline status is displayed in merge requests that use the branch as a source. However, this pipeline type does not support any features offered by -[merge request pipelines](../merge_request_pipelines/), like -[pipelines for merged results](../merge_request_pipelines/pipelines_for_merged_results/index.md) -or [merge trains](../merge_request_pipelines/pipelines_for_merged_results/merge_trains/). +[merge request pipelines](../pipelines/merge_request_pipelines.md), like +[pipelines for merged results](../pipelines/pipelines_for_merged_results.md) +or [merge trains](../pipelines/merge_trains.md). This template intentionally avoids those features. To [include](#include) it: @@ -313,7 +313,7 @@ include: The [`MergeRequest-Pipelines` template](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml) makes your pipelines run for the default branch, tags, and all types of merge request pipelines. Use this template if you use any of the -the [pipelines for merge requests features](../merge_request_pipelines/). +the [pipelines for merge requests features](../pipelines/merge_request_pipelines.md). To [include](#include) it: @@ -1335,7 +1335,7 @@ pipeline based on branch names or pipeline types. | `chat` | For pipelines created by using a [GitLab ChatOps](../chatops/index.md) command. | | `external` | When you use CI services other than GitLab. | | `external_pull_requests` | When an external pull request on GitHub is created or updated (See [Pipelines for external pull requests](../ci_cd_for_external_repos/index.md#pipelines-for-external-pull-requests)). | - | `merge_requests` | For pipelines created when a merge request is created or updated. Enables [merge request pipelines](../merge_request_pipelines/index.md), [merged results pipelines](../merge_request_pipelines/pipelines_for_merged_results/index.md), and [merge trains](../merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). | + | `merge_requests` | For pipelines created when a merge request is created or updated. Enables [merge request pipelines](../pipelines/merge_request_pipelines.md), [merged results pipelines](../pipelines/pipelines_for_merged_results.md), and [merge trains](../pipelines/merge_trains.md). | | `pipelines` | For [multi-project pipelines](../multi_project_pipelines.md) created by [using the API with `CI_JOB_TOKEN`](../multi_project_pipelines.md#create-multi-project-pipelines-by-using-the-api), or the [`trigger`](#trigger) keyword. | | `pushes` | For pipelines triggered by a `git push` event, including for branches and tags. | | `schedules` | For [scheduled pipelines](../pipelines/schedules.md). | @@ -4776,7 +4776,7 @@ pushing multiple changes in a single `git push` invocation. This limitation does not affect any of the updated merge request pipelines. All updated merge requests have a pipeline created when using -[pipelines for merge requests](../merge_request_pipelines/index.md). +[pipelines for merge requests](../pipelines/merge_request_pipelines.md). ## Deprecated keywords diff --git a/doc/development/cicd/index.md b/doc/development/cicd/index.md index 56f511cfcff..e4cac9c62c7 100644 --- a/doc/development/cicd/index.md +++ b/doc/development/cicd/index.md @@ -27,8 +27,8 @@ On the left side we have the events that can trigger a pipeline based on various - A `git push` is the most common event that triggers a pipeline. - The [Web API](../../api/pipelines.md#create-a-new-pipeline). - A user clicking the "Run pipeline" button in the UI. -- When a [merge request is created or updated](../../ci/merge_request_pipelines/index.md#pipelines-for-merge-requests). -- When an MR is added to a [Merge Train](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md#merge-trains). +- When a [merge request is created or updated](../../ci/pipelines/merge_request_pipelines.md#pipelines-for-merge-requests). +- When an MR is added to a [Merge Train](../../ci/pipelines/merge_trains.md#merge-trains). - A [scheduled pipeline](../../ci/pipelines/schedules.md#pipeline-schedules). - When project is [subscribed to an upstream project](../../ci/multi_project_pipelines.md#trigger-a-pipeline-when-an-upstream-project-is-rebuilt). - When [Auto DevOps](../../topics/autodevops/index.md) is enabled. diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 6f1d916fa78..000c909664a 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -422,7 +422,7 @@ WARNING: do not merge the merge request** except for [very specific cases](https://about.gitlab.com/handbook/engineering/workflow/#criteria-for-merging-during-broken-master). For other cases, follow these [handbook instructions](https://about.gitlab.com/handbook/engineering/workflow/#merging-during-broken-master). - - If the **latest [Pipeline for Merged Results](../ci/merge_request_pipelines/pipelines_for_merged_results/#pipelines-for-merged-results)** finished less than 2 hours ago, you + - If the **latest [Pipeline for Merged Results](../ci/pipelines/pipelines_for_merged_results.md)** finished less than 2 hours ago, you might merge without starting a new pipeline as the merge request is close enough to `main`. - When you set the MR to "Merge When Pipeline Succeeds", you should take over @@ -444,7 +444,7 @@ Merge Results against the latest `main` at the time of the pipeline creation. WARNING: **Review all changes thoroughly for malicious code before starting a -[Pipeline for Merged Results](../ci/merge_request_pipelines/index.md#run-pipelines-in-the-parent-project-for-merge-requests-from-a-forked-project).** +[Pipeline for Merged Results](../ci/pipelines/merge_request_pipelines.md#run-pipelines-in-the-parent-project-for-merge-requests-from-a-forked-project).** When reviewing merge requests added by wider community contributors: diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index 9be89202d35..a6dcac47910 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -158,8 +158,8 @@ Commit messages should follow the guidelines below, for reasons explained by Chr Example commit message template that can be used on your machine that embodies the above (guide for [how to apply template](https://codeinthehole.com/tips/a-useful-template-for-commit-messages/)): ```plaintext -# (If applied, this commit will...) (Max 50 char) -# |<---- Using a Maximum Of 50 Characters ---->| +# (If applied, this commit will...) (Max 72 characters) +# |<---- Using a Maximum Of 72 Characters ---->| # Explain why this change is being made diff --git a/doc/development/documentation/index.md b/doc/development/documentation/index.md index b2ac48db7fb..43bcc2843af 100644 --- a/doc/development/documentation/index.md +++ b/doc/development/documentation/index.md @@ -480,7 +480,7 @@ The following GitLab features are used among others: - [Review Apps](../../ci/review_apps/index.md) - [Artifacts](../../ci/yaml/index.md#artifacts) - [Specific runner](../../ci/runners/runners_scope.md#prevent-a-specific-runner-from-being-enabled-for-other-projects) -- [Pipelines for merge requests](../../ci/merge_request_pipelines/index.md) +- [Pipelines for merge requests](../../ci/pipelines/merge_request_pipelines.md) ## Testing diff --git a/doc/development/documentation/structure.md b/doc/development/documentation/structure.md index b2152858e8e..9449fd52839 100644 --- a/doc/development/documentation/structure.md +++ b/doc/development/documentation/structure.md @@ -19,30 +19,6 @@ Each page contains multiple topic types. For example, a page with the title `Pipelines`, which is generated from a file called `index.md`, can include a concept and multiple task and reference topics. -GitLab also uses high-level landing pages. - -## Landing pages - -Landing pages are topics that group other topics and help a user to navigate a section. - -Users who are using the in-product help do not have a left nav, -and need these topics to navigate the documentation. - -These topics can also help other users find the most important topics -in a section. - -Landing page topics should be in this format: - -```markdown -# Title (a noun, like "CI/CD or "Analytics") - -Brief introduction to the concept or product area. -Include the reason why someone would use this thing. - -- Bulleted list of important related topics. -- These links are needed because users of in-product help do not have left navigation. -``` - ## Concept A concept topic introduces a single feature or concept. @@ -248,3 +224,101 @@ Consider the following guidelines when offering examples: Although the bad-then-good approach is acceptable for the GitLab development guidelines, do not use it for user documentation. For user documentation, use *Do* and *Don't*. For examples, see the [Pajamas Design System](https://design.gitlab.com/content/punctuation/). + +## Other types of content + +There are other types of content in the GitLab documentation that don't +classify as one of the four [topic types](#documentation-topic-types). +These include: + +- [Tutorials](#tutorials) +- [Get started pages](#get-started) +- [Topics and resources pages](#topics-and-resources-pages) + +In most cases, these content types are on their own standalone page. + +### Tutorials + +A tutorial is an end-to-end walkthrough of a complex workflow or scenario. +It might include tasks across a variety of GitLab features, tools, and processes. +It does not cover core conceptual information. + +Tutorials should be in this format: + +```markdown +# Title (starts with "Tutorial:" followed by an active verb, like "Tutorial: create a website") + +A paragraph that explains what the tutorial does, and the expected outcome. + +Prerequisites (optional): + +- Thing 1 +- Thing 2 +- Thing 3 + +## Step 1: do the first task + +To do step 1: + +1. First step. +2. Another step. +3. Another step. + +## Step 2: do the second task + +To do step 2: + +1. First step. +2. Another step. +3. Another step. +``` + +### Get started + +A get started page is a set of steps to help a user get set up +quickly to use a single GitLab feature or tool. +It might consist of more than one task. + +Get started pages should be in this format: + +```markdown +# Title ("Get started with ") + +Complete the following steps to ... . + +1. First step. +1. Another step. +1. Another step. + +If you need to add more than one task, +consider using subsections for each distinct task. +``` + +### Topics and resources pages + +This is a page with a list of links that point to important sections +of documentation for a specific GitLab feature or tool. + +We do not encourage the use of these types of pages. +Lists like this can get out of date quickly and offer little value to users. +We've included this type here because: + +- There are existing pages in the documentation that follow this format, + and they should be standardized. +- They can sometimes help navigate a complex section of the documentation. + +If you come across a page like this +or you have to create one, use this format: + +```markdown +# Title ("Topics and resources for ") + +Brief sentence to describe the feature. + +The following topics and resources can help you understand and work with this feature: + +- Link 1 +- Link 2 +- Link 3 + +``` diff --git a/doc/development/documentation/styleguide/index.md b/doc/development/documentation/styleguide/index.md index 4c4e3755451..807e2a12093 100644 --- a/doc/development/documentation/styleguide/index.md +++ b/doc/development/documentation/styleguide/index.md @@ -193,7 +193,7 @@ included in backticks. For example: We include concept and task topic types in the same larger topic. -In general, we have one topic that's a [landing page](../structure.md#landing-pages). +In general, we have one topic that's a landing page. Below that topic in the left nav are individual topics. Each of these include a concept and multiple related tasks, reference, and troubleshooting topics. @@ -750,7 +750,7 @@ For example: | App name | Description | |:---------|:---------------------------------| | App A | Description text. 1 | -| App B | Description text. 2 | +| App B | Description text. 2 | 1. This is the footnote. 1. This is the other footnote. @@ -1531,7 +1531,8 @@ the section. The version information must: - Be surrounded by blank lines. - Start with `>`. If there are multiple bullets, each line must start with `> -`. - The string must include these words in this order (capitalization doesn't matter): - - `introduced`, `deprecated`, `moved` + - `introduced`, `deprecated`, `moved`, `recommended` (as in the + [feature flag documentation](../feature_flags.md)), `removed`, or `renamed` - `in` or `to` - `GitLab` - Whenever possible, include a link to the completed issue, merge request, or epic diff --git a/doc/integration/datadog.md b/doc/integration/datadog.md new file mode 100644 index 00000000000..38ff004a203 --- /dev/null +++ b/doc/integration/datadog.md @@ -0,0 +1,32 @@ +--- +stage: Create +group: Ecosystem +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Datadog integration **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/270123) in GitLab 14.1 + +This integration allows sending CI/CD pipeline and job information to [Datadog](https://www.datadoghq.com/) for monitoring and troubleshooting of job failures and performance issues using the [CI Visibility](https://app.datadoghq.com/ci) product. + +You can find out more information on [Datadog's CI Visibility documentation site](https://docs.datadoghq.com/continuous_integration/). + +## How to configure it + +The integration is based on [Webhooks](../user/project/integrations/webhooks.md) and it only requires setup on GitLab. + +Configure the integration on a project or group by going to **Settings > Integrations > Datadog** for each project or group you want to instrument. You can also activate the integration for the entire GitLab instance. + +Fill in the integration configuration settings: + +- `Active` enables the integration. +- `Datadog site` specifies which [Datadog site](https://docs.datadoghq.com/getting_started/site/) to send data to. +- `API URL` (optional) allows overriding the API URL used for sending data directly, only used in advanced scenarios. +- `API key` specifies which API key to use when sending data. You can generate one in the [APIs tab](https://app.datadoghq.com/account/settings#api) of the Integrations section on Datadog. +- `Service` (optional) specifies which service name to attach to each span generated by the integration. Use this to differentiate between GitLab instances. +- `Env` (optional) specifies which environment (`env` tag) to attach to each span generated by the integration. Use this to differentiate between groups of GitLab instances (i.e. staging vs production). + +You can test the integration with the `Test settings` button. After it’s successful, click `Save changes` to finish the integration set up. + +Data sent by the integration will be available in the [CI Visibility](https://app.datadoghq.com/ci) section of your Datadog account. diff --git a/doc/subscriptions/bronze_starter.md b/doc/subscriptions/bronze_starter.md index b60f80b59fd..e350e989b6e 100644 --- a/doc/subscriptions/bronze_starter.md +++ b/doc/subscriptions/bronze_starter.md @@ -98,7 +98,7 @@ the tiers are no longer mentioned in GitLab documentation: - [Bidirectional mirroring](../user/project/repository/repository_mirroring.md#bidirectional-mirroring) - [Mirroring with Perforce Helix via Git Fusion](../user/project/repository/repository_mirroring.md#mirroring-with-perforce-helix-via-git-fusion) - Runners: - - Run pipelines in the parent project [for merge requests from a forked project](../ci/merge_request_pipelines/index.md#run-pipelines-in-the-parent-project-for-merge-requests-from-a-forked-project) + - Run pipelines in the parent project [for merge requests from a forked project](../ci/pipelines/merge_request_pipelines.md#run-pipelines-in-the-parent-project-for-merge-requests-from-a-forked-project) - [Shared runners pipeline minutes quota](../user/admin_area/settings/continuous_integration.md#shared-runners-pipeline-minutes-quota) - [Push rules](../push_rules/push_rules.md) - SAML for self-managed GitLab instance: diff --git a/doc/topics/git/lfs/index.md b/doc/topics/git/lfs/index.md index dfb175cbb82..32039548475 100644 --- a/doc/topics/git/lfs/index.md +++ b/doc/topics/git/lfs/index.md @@ -275,46 +275,3 @@ You might choose to do this if you are using an appliance like a