From f3af1e76dc7dd36b58cfd13bcdea6efabb79d5ce Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 23 Aug 2024 09:09:28 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- ...Pipeline Authoring Issue Implementation.md | 56 --------- .../Pipeline Authoring Issue.md | 38 +++++++ GITLAB_KAS_VERSION | 2 +- app/models/merge_request.rb | 37 +++--- app/services/issuable_base_service.rb | 5 + app/services/merge_requests/update_service.rb | 40 ++++++- config/initializers/1_settings.rb | 20 ++++ db/docs/ci_builds_runner_session.yml | 1 - db/docs/ci_job_variables.yml | 1 - db/docs/p_ci_job_annotations.yml | 1 - db/docs/p_ci_pipeline_variables.yml | 1 - .../dedicated/hosted_runners.md | 4 +- doc/ci/secrets/index.md | 19 +++- doc/topics/git/troubleshooting_git.md | 107 ++++++++---------- doc/user/gitlab_duo/index.md | 8 +- lib/gitlab/middleware/path_traversal_check.rb | 16 ++- lib/gitlab/observability.rb | 4 + lib/gitlab_settings/options.rb | 4 + locale/gitlab.pot | 3 - qa/qa/runtime/browser.rb | 2 + .../components/related_issues_block_spec.js | 33 ++++++ spec/lib/gitlab/database/sharding_key_spec.rb | 5 +- .../middleware/path_traversal_check_spec.rb | 49 ++++++-- spec/lib/gitlab_settings/options_spec.rb | 19 ++++ spec/models/merge_request_spec.rb | 49 -------- .../merge_requests/update_service_spec.rb | 51 ++++++++- 26 files changed, 347 insertions(+), 228 deletions(-) delete mode 100644 .gitlab/issue_templates/Pipeline Authoring Issue Implementation.md create mode 100644 .gitlab/issue_templates/Pipeline Authoring Issue.md diff --git a/.gitlab/issue_templates/Pipeline Authoring Issue Implementation.md b/.gitlab/issue_templates/Pipeline Authoring Issue Implementation.md deleted file mode 100644 index 08049daab22..00000000000 --- a/.gitlab/issue_templates/Pipeline Authoring Issue Implementation.md +++ /dev/null @@ -1,56 +0,0 @@ - - -## Summary - -## Proposal - -## Confirm purpose and User Reception (how does this benefit the user?) - -## Additional details - - -Some relevant technical details, if applicable, such as: - -- Does this need a ~"feature flag"? -- Does there need to be an associated ~"instrumentation" issue created related to this work? -- Is there an example response showing the data structure that should be returned (new endpoints only)? -- What permissions should be used? -- Is this EE or CE? - - [ ] EE - - [ ] CE -- Additional comments: - -## Implementation Table - - - - -| Group | Issue Link | -| ------ | ------ | -| ~backend | :point_left: You are here | -| ~frontend | [#123123](url) | - - - -## Links/References - - - - -/label ~"group::pipeline authoring" ~"Category:Pipeline Composition" ~"section::ci" ~"devops::verify" ~"workflow::planning breakdown" diff --git a/.gitlab/issue_templates/Pipeline Authoring Issue.md b/.gitlab/issue_templates/Pipeline Authoring Issue.md new file mode 100644 index 00000000000..88b53efc47e --- /dev/null +++ b/.gitlab/issue_templates/Pipeline Authoring Issue.md @@ -0,0 +1,38 @@ +## Summary + + +## Definition of Done + + + + + +## Details + + +## Roles involved + +- [ ] Design +- [ ] Technical Writing +- [ ] Backend +- [ ] Frontend +- [ ] Test engineering + +**NOTE:** When work is ready to be defined, please create appropriate sub-tasks. + +/label ~"group::pipeline authoring" ~"Category:Pipeline Composition" ~"section::ci" ~"devops::verify" diff --git a/GITLAB_KAS_VERSION b/GITLAB_KAS_VERSION index 65a154b8d3a..df71bf30975 100644 --- a/GITLAB_KAS_VERSION +++ b/GITLAB_KAS_VERSION @@ -1 +1 @@ -1a16fa05c2645a0abba4e2f028e1fdbe5d85be2f +dcb656ee580c8503bf25c06dd6ee117178dcb2a1 diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index a5bba3fd986..2fb74c06bea 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1463,9 +1463,6 @@ class MergeRequest < ApplicationRecord def cache_merge_request_closes_issues!(current_user = self.author) return if closed? || merged? - issue_ids_existing = merge_requests_closing_issues - .from_mr_description - .pluck(:issue_id) issues_to_close_ids = closes_issues(current_user).reject { |issue| issue.is_a?(ExternalIssue) }.map(&:id) transaction do @@ -1481,29 +1478,23 @@ class MergeRequest < ApplicationRecord end issue_ids_to_create = issues_to_close_ids - issue_ids_to_update + next unless issue_ids_to_create.any? - if issue_ids_to_create.any? - now = Time.zone.now - new_associations = issue_ids_to_create.map do |issue_id| - MergeRequestsClosingIssues.new( - issue_id: issue_id, - merge_request_id: id, - from_mr_description: true, - created_at: now, - updated_at: now - ) - end - - # We can't skip validations here in bulk insert as we don't have a unique constraint on the DB. - # We can skip validations once we have validated the unique constraint - # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/456965 - MergeRequestsClosingIssues.bulk_insert!(new_associations, batch_size: 100) + now = Time.zone.now + new_associations = issue_ids_to_create.map do |issue_id| + MergeRequestsClosingIssues.new( + issue_id: issue_id, + merge_request_id: id, + from_mr_description: true, + created_at: now, + updated_at: now + ) end - end - ids_for_trigger = (issue_ids_existing + issues_to_close_ids).uniq - WorkItem.id_in(ids_for_trigger).find_each(batch_size: 100) do |work_item| - GraphqlTriggers.work_item_updated(work_item) + # We can't skip validations here in bulk insert as we don't have a unique constraint on the DB. + # We can skip validations once we have validated the unique constraint + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/456965 + MergeRequestsClosingIssues.bulk_insert!(new_associations, batch_size: 100) end end diff --git a/app/services/issuable_base_service.rb b/app/services/issuable_base_service.rb index 8b20bffc4b0..a8b0dbf967d 100644 --- a/app/services/issuable_base_service.rb +++ b/app/services/issuable_base_service.rb @@ -384,9 +384,14 @@ class IssuableBaseService < ::BaseContainerService end end + trigger_update_subscriptions(issuable, old_associations) + issuable end + # Overriden in child class + def trigger_update_subscriptions(issuable, old_associations); end + def transaction_update(issuable, opts = {}) touch = opts[:save_with_touch] || false diff --git a/app/services/merge_requests/update_service.rb b/app/services/merge_requests/update_service.rb index 07a5d77a2fd..ae937daffca 100644 --- a/app/services/merge_requests/update_service.rb +++ b/app/services/merge_requests/update_service.rb @@ -69,15 +69,51 @@ module MergeRequests MergeRequests::CloseService end - def after_update(issuable, old_associations) + def after_update(merge_request, old_associations) super - issuable.cache_merge_request_closes_issues!(current_user) + + merge_request.cache_merge_request_closes_issues!(current_user) + @trigger_work_item_updated = true end private attr_reader :target_branch_was_deleted + def trigger_updated_work_item_on_closing_issues(merge_request, old_closing_issues_ids) + new_issue_ids = merge_request.merge_requests_closing_issues.limit(1000).pluck(:issue_id) # rubocop:disable CodeReuse/ActiveRecord -- Implementation would be the same in the model + all_issue_ids = new_issue_ids | old_closing_issues_ids + return if all_issue_ids.blank? + + WorkItem.id_in(all_issue_ids).find_each(batch_size: 100) do |work_item| # rubocop:disable CodeReuse/ActiveRecord -- Implementation would be the same in the model + GraphqlTriggers.work_item_updated(work_item) + end + end + + override :associations_before_update + def associations_before_update(merge_request) + super.merge( + closing_issues_ids: merge_request.merge_requests_closing_issues.limit(1000).pluck(:issue_id) # rubocop:disable CodeReuse/ActiveRecord -- Implementation would be the same in the model + ) + end + + override :change_state + def change_state(merge_request) + return unless super + + @trigger_work_item_updated = true + end + + override :trigger_update_subscriptions + def trigger_update_subscriptions(merge_request, old_associations) + return unless @trigger_work_item_updated + + trigger_updated_work_item_on_closing_issues( + merge_request, + old_associations.fetch(:closing_issues_ids, []) + ) + end + def general_fallback(merge_request) # We don't allow change of source/target projects and source branch # after merge request was created diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index b1dffc104f4..8377fab3243 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -930,6 +930,9 @@ Gitlab.ee do Settings.cron_jobs['gitlab_subscriptions_add_on_purchases_cleanup_worker'] ||= {} Settings.cron_jobs['gitlab_subscriptions_add_on_purchases_cleanup_worker']['cron'] ||= '0 1 * * *' Settings.cron_jobs['gitlab_subscriptions_add_on_purchases_cleanup_worker']['job_class'] = 'GitlabSubscriptions::AddOnPurchases::CleanupWorker' + Settings.cron_jobs['observability_alert_query_worker'] ||= {} + Settings.cron_jobs['observability_alert_query_worker']['cron'] ||= '* * * * *' + Settings.cron_jobs['observability_alert_query_worker']['job_class'] = 'Observability::AlertQueryWorker' Gitlab.com do Settings.cron_jobs['disable_legacy_open_source_license_for_inactive_projects'] ||= {} @@ -1022,6 +1025,23 @@ Gitlab.ee do Settings.cloud_connector['base_url'] ||= ENV['CLOUD_CONNECTOR_BASE_URL'] || 'https://cloud.gitlab.com' end +# +# Duo Workflow +# +Gitlab.ee do + Settings['duo_workflow'] ||= {} + Settings.duo_workflow.reverse_merge!( + secure: true + ) + + # Default to proxy via Cloud Connector + unless Settings.duo_workflow['service_url'].present? + cloud_connector_uri = URI.parse(Settings.cloud_connector.base_url) + Settings.duo_workflow['service_url'] = "#{cloud_connector_uri.host}:#{cloud_connector_uri.port}" + Settings.duo_workflow['secure'] = cloud_connector_uri.scheme == 'https' + end +end + # # Zoekt credentials # diff --git a/db/docs/ci_builds_runner_session.yml b/db/docs/ci_builds_runner_session.yml index 393ba07fb08..e527de2b380 100644 --- a/db/docs/ci_builds_runner_session.yml +++ b/db/docs/ci_builds_runner_session.yml @@ -8,7 +8,6 @@ description: Store build-related runner session. Data is removed after the respe introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6208 milestone: '11.1' gitlab_schema: gitlab_ci -sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/459994 desired_sharding_key: project_id: references: projects diff --git a/db/docs/ci_job_variables.yml b/db/docs/ci_job_variables.yml index 3903e8f3077..ed7ebc94518 100644 --- a/db/docs/ci_job_variables.yml +++ b/db/docs/ci_job_variables.yml @@ -8,7 +8,6 @@ description: CI/CD variables set to a job when running it manually. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/14784 milestone: '12.2' gitlab_schema: gitlab_ci -sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/463243 desired_sharding_key: project_id: references: projects diff --git a/db/docs/p_ci_job_annotations.yml b/db/docs/p_ci_job_annotations.yml index ef71ffed3dd..45c8087c4e1 100644 --- a/db/docs/p_ci_job_annotations.yml +++ b/db/docs/p_ci_job_annotations.yml @@ -9,7 +9,6 @@ description: Stores user provided annotations for jobs. Currently storing extra introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117319 milestone: '16.1' gitlab_schema: gitlab_ci -sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/463246 desired_sharding_key: project_id: references: projects diff --git a/db/docs/p_ci_pipeline_variables.yml b/db/docs/p_ci_pipeline_variables.yml index 7cd0139fe7d..97063637cb6 100644 --- a/db/docs/p_ci_pipeline_variables.yml +++ b/db/docs/p_ci_pipeline_variables.yml @@ -8,7 +8,6 @@ description: Routing table for ci_pipeline_variables introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/141270 milestone: '16.9' gitlab_schema: gitlab_ci -sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/463244 desired_sharding_key: project_id: references: projects diff --git a/doc/administration/dedicated/hosted_runners.md b/doc/administration/dedicated/hosted_runners.md index 71ec9648bf7..d212e0dd568 100644 --- a/doc/administration/dedicated/hosted_runners.md +++ b/doc/administration/dedicated/hosted_runners.md @@ -1,6 +1,6 @@ --- -stage: SaaS Platforms -group: GitLab Dedicated +stage: Verify +group: Hosted Runners description: Use hosted runners to run your CI/CD jobs on GitLab Dedicated. info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/ci/secrets/index.md b/doc/ci/secrets/index.md index 5c59664597a..8ff70885c78 100644 --- a/doc/ci/secrets/index.md +++ b/doc/ci/secrets/index.md @@ -54,6 +54,8 @@ tutorial for a version of this feature. It's available to all subscription levels, supports writing secrets to and deleting secrets from Vault, and supports multiple secrets engines. +You must replace the `vault.example.com` URL below with the URL of your Vault server, and `gitlab.example.com` with the URL of your GitLab instance. + ## Vault Secrets Engines > - `generic` option [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/366492) in GitLab Runner 16.11. @@ -108,9 +110,13 @@ To configure your Vault server: If no role is specified, Vault uses the [default role](https://developer.hashicorp.com/vault/api-docs/auth/jwt#default_role) specified when the authentication method was configured. - `VAULT_AUTH_PATH` - Optional. The path where the authentication method is mounted, default is `jwt`. - - `VAULT_NAMESPACE` - Optional. The [Vault Enterprise namespace](https://developer.hashicorp.com/vault/docs/enterprise/namespaces) to use for reading secrets and authentication. - If no namespace is specified, Vault uses the `root` ("`/`") namespace. - The setting is ignored by Vault Open Source. + - `VAULT_NAMESPACE` - Optional. The [Vault Enterprise namespace](https://developer.hashicorp.com/vault/docs/enterprise/namespaces) + to use for reading secrets and authentication. With: + - Vault, the `root` ("`/`") namespace is used when no namespace is specified. + - Vault Open source, the setting is ignored. + - [HashiCorp Cloud Platform (HCP)](https://www.hashicorp.com/cloud) Vault, a namespace + is required. HCP Vault uses the `admin` namespace as the root namespace by default. + For example, `VAULT_NAMESPACE=admin`. NOTE: Support for providing these values in the user interface [is tracked in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/218677). @@ -128,7 +134,7 @@ the secrets stored in Vault by defining them with the [`vault` keyword](../yaml/ job_using_vault: id_tokens: VAULT_ID_TOKEN: - aud: https://gitlab.com + aud: https://vault.example.com secrets: DATABASE_PASSWORD: vault: production/db/password@ops # translates to secret `ops/data/production/db`, field `password` @@ -151,7 +157,7 @@ To overwrite the default behavior, set the `file` option explicitly: secrets: id_tokens: VAULT_ID_TOKEN: - aud: https://gitlab.com + aud: https://vault.example.com DATABASE_PASSWORD: vault: production/db/password@ops file: false @@ -172,7 +178,7 @@ For example, to set the secret engine and path for Artifactory: job_using_vault: id_tokens: VAULT_ID_TOKEN: - aud: https://gitlab.com + aud: https://vault.example.com secrets: JFROG_TOKEN: vault: @@ -212,6 +218,7 @@ $ vault write auth/jwt/role/myproject-production - <