diff --git a/Gemfile b/Gemfile index fedc9649978..f08b2bc2e48 100644 --- a/Gemfile +++ b/Gemfile @@ -463,6 +463,8 @@ group :development, :test do # See: https://gitlab.com/gitlab-org/frontend/rfcs/-/issues/106 gem 'vite_rails', '~> 3.0.17', feature_category: :shared gem 'vite_ruby', '~> 3.5.0', feature_category: :shared + + gem 'gitlab-housekeeper', path: 'gems/gitlab-housekeeper' # rubocop:todo Gemfile/MissingFeatureCategory end group :development, :test, :danger do diff --git a/Gemfile.lock b/Gemfile.lock index e66433637df..cf561326616 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -29,6 +29,13 @@ PATH gitlab-backup-cli (0.0.1) thor (~> 1.3) +PATH + remote: gems/gitlab-housekeeper + specs: + gitlab-housekeeper (0.1.0) + httparty + rubocop + PATH remote: gems/gitlab-http specs: @@ -1877,6 +1884,7 @@ DEPENDENCIES gitlab-dangerfiles (~> 4.6.0) gitlab-experiment (~> 0.9.1) gitlab-fog-azure-rm (~> 1.8.0) + gitlab-housekeeper! gitlab-http! gitlab-labkit (~> 0.34.0) gitlab-license (~> 2.3) diff --git a/app/assets/javascripts/super_sidebar/components/help_center.vue b/app/assets/javascripts/super_sidebar/components/help_center.vue index 4e0d05add85..5278bd66f47 100644 --- a/app/assets/javascripts/super_sidebar/components/help_center.vue +++ b/app/assets/javascripts/super_sidebar/components/help_center.vue @@ -251,7 +251,7 @@ export default { diff --git a/app/models/integration.rb b/app/models/integration.rb index c7161574a17..618f9f986e8 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -526,6 +526,17 @@ class Integration < ApplicationRecord fields.reject { _1[:type] == :password || _1[:name] == 'webhook' || (_1.key?(:if) && _1[:if] != true) }.pluck(:name) end + def self.api_fields + fields.map do |field| + { + required: field.required?, + name: field.name.to_sym, + type: field.api_type, + desc: field.description + } + end + end + def form_fields fields.reject { _1[:api_only] == true || (_1.key?(:if) && _1[:if] != true) } end diff --git a/app/models/integrations/apple_app_store.rb b/app/models/integrations/apple_app_store.rb index f8fddf8a457..a248a1aa561 100644 --- a/app/models/integrations/apple_app_store.rb +++ b/app/models/integrations/apple_app_store.rb @@ -21,21 +21,31 @@ module Integrations field :app_store_issuer_id, section: SECTION_TYPE_CONNECTION, required: true, - title: -> { s_('AppleAppStore|The Apple App Store Connect Issuer ID.') } + title: -> { s_('AppleAppStore|Apple App Store Connect issuer ID') }, + description: -> { s_('AppleAppStore|Apple App Store Connect issuer ID.') } field :app_store_key_id, section: SECTION_TYPE_CONNECTION, required: true, - title: -> { s_('AppleAppStore|The Apple App Store Connect Key ID.') } + title: -> { s_('AppleAppStore|Apple App Store Connect key ID') }, + description: -> { s_('AppleAppStore|Apple App Store Connect key ID.') } - field :app_store_private_key_file_name, section: SECTION_TYPE_CONNECTION - field :app_store_private_key, api_only: true + field :app_store_private_key_file_name, + description: -> { s_('Apple App Store Connect private key file name.') }, + section: SECTION_TYPE_CONNECTION, + required: true + + field :app_store_private_key, + description: -> { s_('Apple App Store Connect private key.') }, + required: true, + api_only: true field :app_store_protected_refs, type: :checkbox, section: SECTION_TYPE_CONFIGURATION, title: -> { s_('AppleAppStore|Protected branches and tags only') }, - checkbox_label: -> { s_('AppleAppStore|Only set variables on protected branches and tags') } + description: -> { s_('AppleAppStore|Set variables on protected branches and tags only.') }, + checkbox_label: -> { s_('AppleAppStore|Set variables on protected branches and tags only.') } def self.title 'Apple App Store Connect' @@ -55,10 +65,10 @@ module Integrations # rubocop:disable Layout/LineLength texts = [ - s_("Use the Apple App Store Connect integration to easily connect to the Apple App Store with Fastlane in CI/CD pipelines."), - s_("After the Apple App Store Connect integration is activated, the following protected variables will be created for CI/CD use."), + s_("Use this integration to connect to the Apple App Store with fastlane in CI/CD pipelines."), + s_("After you enable the integration, the following protected variables are created for CI/CD use:"), variable_list.join('
'), - s_(format("To get started, see the integration documentation for instructions on how to generate App Store Connect credentials, and how to use this integration.", url: Rails.application.routes.url_helpers.help_page_url('user/project/integrations/apple_app_store'))).html_safe + s_(format("For more information, see the documentation.", url: Rails.application.routes.url_helpers.help_page_url('user/project/integrations/apple_app_store'))).html_safe ] # rubocop:enable Layout/LineLength diff --git a/app/models/integrations/field.rb b/app/models/integrations/field.rb index 9dc90629344..324848daf4f 100644 --- a/app/models/integrations/field.rb +++ b/app/models/integrations/field.rb @@ -6,7 +6,7 @@ module Integrations ATTRIBUTES = %i[ section type placeholder choices value checkbox_label - title help if + title help if description non_empty_password_help non_empty_password_title ].concat(BOOLEAN_ATTRIBUTES).freeze @@ -60,6 +60,10 @@ module Integrations define_method("#{type}?") { self[:type] == type } end + def api_type + checkbox? ? ::API::Integrations::Boolean : String + end + private attr_reader :attributes diff --git a/config/feature_flags/experiment/use_cloud_connector_lb.yml b/config/feature_flags/experiment/use_cloud_connector_lb.yml new file mode 100644 index 00000000000..e77c333d214 --- /dev/null +++ b/config/feature_flags/experiment/use_cloud_connector_lb.yml @@ -0,0 +1,8 @@ +--- +name: use_cloud_connector_lb +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139265 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/435142 +milestone: '16.8' +type: experiment +group: group::cloud connector +default_enabled: false diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 974e8cce284..ade5465f8ea 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -954,6 +954,14 @@ Gitlab.ee do Settings.suggested_reviewers['secret_file'] ||= Rails.root.join('.gitlab_suggested_reviewers_secret') end +# +# Cloud connector +# +Gitlab.ee do + Settings['cloud_connector'] = {} + Settings.cloud_connector['base_url'] ||= ENV['CLOUD_CONNECTOR_BASE_URL'] || 'https://cloud.gitlab.com' +end + # # Zoekt credentials # diff --git a/config/locales/doorkeeper.en.yml b/config/locales/doorkeeper.en.yml index e4ecc4bbb83..dbbbbf86994 100644 --- a/config/locales/doorkeeper.en.yml +++ b/config/locales/doorkeeper.en.yml @@ -134,7 +134,7 @@ en: write_observability: Grants write access to GitLab Observability. ai_features: - Grants permission to perform API actions for GitLab Duo. + Grants permission to perform API actions for GitLab Duo. This scope is designed to work with the GitLab Duo Plugin for JetBrains. For all other extensions, see scope requirements. openid: Grants permission to authenticate with GitLab using OpenID Connect. Also gives read-only access to the user's profile and group memberships. sudo: diff --git a/doc/api/integrations.md b/doc/api/integrations.md index 6821858234f..a1f504a3fd8 100644 --- a/doc/api/integrations.md +++ b/doc/api/integrations.md @@ -78,11 +78,11 @@ Example response: ] ``` -## Apple App Store +## Apple App Store Connect -### Set up Apple App Store +### Set up Apple App Store Connect -Set up the Apple App Store integration for a project. +Set up the Apple App Store Connect integration for a project. ```plaintext PUT /projects/:id/integrations/apple_app_store @@ -92,10 +92,11 @@ Parameters: | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `app_store_issuer_id` | string | true | The Apple App Store Connect Issuer ID. | -| `app_store_key_id` | string | true | The Apple App Store Connect Key ID. | -| `app_store_private_key` | string | true | The Apple App Store Connect Private Key. | -| `app_store_protected_refs` | boolean | false | Set variables only on protected branches and tags. Defaults to `true` (enabled). | +| `app_store_issuer_id` | string | true | Apple App Store Connect issuer ID. | +| `app_store_key_id` | string | true | Apple App Store Connect key ID. | +| `app_store_private_key_file_name` | string | true | Apple App Store Connect private key file name. | +| `app_store_private_key` | string | true | Apple App Store Connect private key. | +| `app_store_protected_refs` | boolean | false | Set variables on protected branches and tags only. | ### Disable Apple App Store @@ -677,7 +678,7 @@ Parameters: | --------- | ---- | -------- | ----------- | | `token` | string | true | GitHub API token with `repo:status` OAuth scope. | | `repository_url` | string | true | GitHub repository URL. | -| `static_context` | boolean | false | Append instance name instead of branch to [status check name](../user/project/integrations/github.md#static-or-dynamic-status-check-names). | +| `static_context` | boolean | false | Append the hostname of your GitLab instance to the [status check name](../user/project/integrations/github.md#static-or-dynamic-status-check-names). | ### Disable GitHub diff --git a/doc/user/group/settings/group_access_tokens.md b/doc/user/group/settings/group_access_tokens.md index 2d77e78e91f..5b3f962061e 100644 --- a/doc/user/group/settings/group_access_tokens.md +++ b/doc/user/group/settings/group_access_tokens.md @@ -152,7 +152,7 @@ The scope determines the actions you can perform when you authenticate with a gr | `read_repository` | Grants read access (pull) to all repositories within a group. | | `write_repository` | Grants read and write access (pull and push) to all repositories within a group. | | `create_runner` | Grants permission to create runners in a group. | -| `ai_features` | Grants permission to perform API actions for GitLab Duo. | +| `ai_features` | Grants permission to perform API actions for GitLab Duo. This scope is designed to work with the GitLab Duo Plugin for JetBrains. For all other extensions, see scope requirements. | | `k8s_proxy` | Grants permission to perform Kubernetes API calls using the agent for Kubernetes in a group. | ## Enable or disable group access token creation diff --git a/doc/user/profile/personal_access_tokens.md b/doc/user/profile/personal_access_tokens.md index ac6ec4ce2f0..9d54381ef87 100644 --- a/doc/user/profile/personal_access_tokens.md +++ b/doc/user/profile/personal_access_tokens.md @@ -123,7 +123,7 @@ A personal access token can perform actions based on the assigned scopes. | `sudo` | Grants permission to perform API actions as any user in the system, when authenticated as an administrator. | | `admin_mode` | Grants permission to perform API actions as an administrator, when Admin Mode is enabled. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/107875) in GitLab 15.8.) | | `create_runner` | Grants permission to create runners. | -| `ai_features` | Grants permission to perform API actions for GitLab Duo. | +| `ai_features` | Grants permission to perform API actions for GitLab Duo. This scope is designed to work with the GitLab Duo Plugin for JetBrains. For all other extensions, see scope requirements. | | `k8s_proxy` | Grants permission to perform Kubernetes API calls using the agent for Kubernetes. | WARNING: diff --git a/doc/user/project/integrations/apple_app_store.md b/doc/user/project/integrations/apple_app_store.md index fdfed554bd8..3031ae42e4d 100644 --- a/doc/user/project/integrations/apple_app_store.md +++ b/doc/user/project/integrations/apple_app_store.md @@ -4,7 +4,7 @@ group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Apple App Store **(FREE ALL)** +# Apple App Store Connect **(FREE ALL)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104888) in GitLab 15.8 [with a flag](../../../administration/feature_flags.md) named `apple_app_store_integration`. Disabled by default. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/385335) in GitLab 15.10. Feature flag `apple_app_store_integration` removed. @@ -16,9 +16,9 @@ The feature is still in development, but you can: - [Report a bug](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=report_bug). - [Share feedback](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=general_feedback). -With the Apple App Store integration, you can configure your CI/CD pipelines to connect to [App Store Connect](https://appstoreconnect.apple.com) to build and release apps for iOS, iPadOS, macOS, tvOS, and watchOS. +With the Apple App Store Connect integration, you can configure your CI/CD pipelines to connect to [App Store Connect](https://appstoreconnect.apple.com) to build and release apps for iOS, iPadOS, macOS, tvOS, and watchOS. -The Apple App Store integration works out of the box with [fastlane](https://fastlane.tools/). You can also use this integration with other build tools. +The Apple App Store Connect integration works out of the box with [fastlane](https://fastlane.tools/). You can also use this integration with other build tools. ## Prerequisites @@ -26,7 +26,7 @@ An Apple ID enrolled in the [Apple Developer Program](https://developer.apple.co ## Configure GitLab -GitLab supports enabling the Apple App Store integration at the project level. Complete these steps in GitLab: +GitLab supports enabling the Apple App Store Connect integration at the project level. Complete these steps in GitLab: 1. In the Apple App Store Connect portal, generate a new private key for your project by following [these instructions](https://developer.apple.com/documentation/appstoreconnectapi/creating_api_keys_for_app_store_connect_api). 1. On the left sidebar, select **Search or go to** and find your project. @@ -36,15 +36,15 @@ GitLab supports enabling the Apple App Store integration at the project level. C 1. Provide the Apple App Store Connect configuration information: - **Issuer ID**: The Apple App Store Connect issuer ID. - **Key ID**: The key ID of the generated private key. - - **Private Key**: The generated private key. You can download this key only once. - - **Protected branches and tags only**: Enable to only set variables on protected branches and tags. + - **Private key**: The generated private key. You can download this key only once. + - **Protected branches and tags only**: Enable to set variables on protected branches and tags only. 1. Select **Save changes**. -After the Apple App Store integration is activated: +After you enable the integration: - The global variables `$APP_STORE_CONNECT_API_KEY_ISSUER_ID`, `$APP_STORE_CONNECT_API_KEY_KEY_ID`, `$APP_STORE_CONNECT_API_KEY_KEY`, and `$APP_STORE_CONNECT_API_KEY_IS_KEY_CONTENT_BASE64` are created for CI/CD use. -- `$APP_STORE_CONNECT_API_KEY_KEY` contains the Base64 encoded Private Key. +- `$APP_STORE_CONNECT_API_KEY_KEY` contains the Base64-encoded private key. - `$APP_STORE_CONNECT_API_KEY_IS_KEY_CONTENT_BASE64` is always `true`. ## Security considerations @@ -52,7 +52,7 @@ After the Apple App Store integration is activated: ### CI/CD variable security Malicious code pushed to your `.gitlab-ci.yml` file could compromise your variables, including -`$APP_STORE_CONNECT_API_KEY_KEY`, and send them to a third-party server. For more details, see +`$APP_STORE_CONNECT_API_KEY_KEY`, and send them to a third-party server. For more information, see [CI/CD variable security](../../../ci/variables/index.md#cicd-variable-security). ## Enable the integration in fastlane diff --git a/doc/user/project/merge_requests/approvals/settings.md b/doc/user/project/merge_requests/approvals/settings.md index a6d24234abf..f9e40a6714c 100644 --- a/doc/user/project/merge_requests/approvals/settings.md +++ b/doc/user/project/merge_requests/approvals/settings.md @@ -114,7 +114,8 @@ When this field is changed, it can affect all open merge requests depending on t ## Require user re-authentication to approve -> Requiring re-authentication by using SAML authentication for GitLab.com groups [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5981) in GitLab 16.6 [with a flag](../../../../administration/feature_flags.md) named `ff_require_saml_auth_to_approve`. Disabled by default. +> - Requiring re-authentication by using SAML authentication for GitLab.com groups [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5981) in GitLab 16.6 [with a flag](../../../../administration/feature_flags.md) named `ff_require_saml_auth_to_approve`. Disabled by default. +> - Requiring re-authentication by using SAML authentication for self-managed instances [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/431415) in GitLab 16.7 [with a flag](../../../../administration/feature_flags.md) named `ff_require_saml_auth_to_approve`. Disabled by default. FLAG: On self-managed GitLab, by default requiring re-authentication by using SAML authentication is not available. To make it available, an administrator can @@ -123,16 +124,18 @@ On self-managed GitLab, by default requiring re-authentication by using SAML aut You can force potential approvers to first authenticate with either: - A password. -- SAML. Available on GitLab.com groups only. +- SAML. This permission enables an electronic signature for approvals, such as the one defined by -[Code of Federal Regulations (CFR) Part 11](https://www.accessdata.fda.gov/scripts/cdrh/cfdocs/cfcfr/CFRSearch.cfm?CFRPart=11&showFR=1&subpartNode=21:1.0.1.1.8.3)): +[Code of Federal Regulations (CFR) Part 11](https://www.accessdata.fda.gov/scripts/cdrh/cfdocs/cfcfr/CFRSearch.cfm?CFRPart=11&showFR=1&subpartNode=21:1.0.1.1.8.3)). This +setting is only available on top-level groups. For more information, see [Settings cascading](#settings-cascading). 1. Enable password authentication and SAML authentication. For more information on: - Password authentication, see [sign-in restrictions documentation](../../../../administration/settings/sign_in_restrictions.md#password-authentication-enabled). - SAML authentication for GitLab.com groups, see [SAML SSO for GitLab.com groups documentation](../../../../user/group/saml_sso). + - SAML authentication for self-managed instances, see [SAML SSO for self-managed GitLab instances](../../../../integration/saml.md). 1. On the left sidebar, select **Settings > Merge requests**. 1. In the **Merge request approvals** section, scroll to **Approval settings** and select **Require user re-authentication (password or SAML) to approve**. diff --git a/doc/user/project/settings/project_access_tokens.md b/doc/user/project/settings/project_access_tokens.md index 393091b43ac..f22fe603f81 100644 --- a/doc/user/project/settings/project_access_tokens.md +++ b/doc/user/project/settings/project_access_tokens.md @@ -96,7 +96,7 @@ See the warning in [create a project access token](#create-a-project-access-toke | `read_repository` | Grants read access (pull) to the repository. | | `write_repository` | Grants read and write access (pull and push) to the repository. | | `create_runner` | Grants permission to create runners in the project. | -| `ai_features` | Grants permission to perform API actions for GitLab Duo. | +| `ai_features` | Grants permission to perform API actions for GitLab Duo. This scope is designed to work with the GitLab Duo Plugin for JetBrains. For all other extensions, see scope requirements. | | `k8s_proxy` | Grants permission to perform Kubernetes API calls using the agent for Kubernetes in the project. | ## Enable or disable project access token creation diff --git a/gems/gitlab-housekeeper/.gitignore b/gems/gitlab-housekeeper/.gitignore new file mode 100644 index 00000000000..48cc0651718 --- /dev/null +++ b/gems/gitlab-housekeeper/.gitignore @@ -0,0 +1,12 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ + +# rspec failure tracking +.rspec_status +/*.gem diff --git a/gems/gitlab-housekeeper/.gitlab-ci.yml b/gems/gitlab-housekeeper/.gitlab-ci.yml new file mode 100644 index 00000000000..2d7f34238a2 --- /dev/null +++ b/gems/gitlab-housekeeper/.gitlab-ci.yml @@ -0,0 +1,4 @@ +include: + - local: gems/gem.gitlab-ci.yml + inputs: + gem_name: "gitlab-housekeeper" diff --git a/gems/gitlab-housekeeper/.rspec b/gems/gitlab-housekeeper/.rspec new file mode 100644 index 00000000000..34c5164d9b5 --- /dev/null +++ b/gems/gitlab-housekeeper/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/gems/gitlab-housekeeper/.rubocop.yml b/gems/gitlab-housekeeper/.rubocop.yml new file mode 100644 index 00000000000..ebe21242469 --- /dev/null +++ b/gems/gitlab-housekeeper/.rubocop.yml @@ -0,0 +1,14 @@ +inherit_from: + - ../config/rubocop.yml + +Gitlab/HTTParty: + Enabled: false + +Rails/Output: + Enabled: false + +Gemfile/MissingFeatureCategory: + Enabled: false + +Rails/NegateInclude: + Enabled: false diff --git a/gems/gitlab-housekeeper/Gemfile b/gems/gitlab-housekeeper/Gemfile new file mode 100644 index 00000000000..88ede059510 --- /dev/null +++ b/gems/gitlab-housekeeper/Gemfile @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +# Specify your gem's dependencies in gitlab-housekeeper.gemspec +gemspec + +gem "rake", "~> 13.0" + +gem "rspec", "~> 3.0" +gem "pry" +gem 'webmock' + +group :development, :test do + gem 'gitlab-rspec', path: '../gitlab-rspec' +end diff --git a/gems/gitlab-housekeeper/Gemfile.lock b/gems/gitlab-housekeeper/Gemfile.lock new file mode 100644 index 00000000000..2b18c02558c --- /dev/null +++ b/gems/gitlab-housekeeper/Gemfile.lock @@ -0,0 +1,139 @@ +PATH + remote: ../gitlab-rspec + specs: + gitlab-rspec (0.1.0) + activerecord (>= 6.1, < 8) + activesupport (>= 6.1, < 8) + rspec (~> 3.0) + +PATH + remote: . + specs: + gitlab-housekeeper (0.1.0) + httparty + rubocop + +GEM + remote: https://rubygems.org/ + specs: + activemodel (7.0.8) + activesupport (= 7.0.8) + activerecord (7.0.8) + activemodel (= 7.0.8) + activesupport (= 7.0.8) + activesupport (7.0.8) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + addressable (2.8.5) + public_suffix (>= 2.0.2, < 6.0) + ast (2.4.2) + coderay (1.1.3) + concurrent-ruby (1.2.2) + crack (0.4.3) + safe_yaml (~> 1.0.0) + diff-lcs (1.5.0) + gitlab-styles (10.1.0) + rubocop (~> 1.50.2) + rubocop-graphql (~> 0.18) + rubocop-performance (~> 1.15) + rubocop-rails (~> 2.17) + rubocop-rspec (~> 2.22) + hashdiff (1.0.1) + httparty (0.21.0) + mini_mime (>= 1.0.0) + multi_xml (>= 0.5.2) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + json (2.7.1) + method_source (1.0.0) + mini_mime (1.1.5) + minitest (5.20.0) + multi_xml (0.6.0) + parallel (1.23.0) + parser (3.2.2.4) + ast (~> 2.4.1) + racc + pry (0.14.2) + coderay (~> 1.1) + method_source (~> 1.0) + public_suffix (5.0.3) + racc (1.7.3) + rack (3.0.8) + rainbow (3.1.1) + rake (13.1.0) + regexp_parser (2.8.3) + rexml (3.2.6) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.6) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-rails (1.3.2) + rack (>= 1.0.0) + rspec (>= 1.3.0) + rspec-support (3.12.1) + rubocop (1.50.2) + json (~> 2.3) + parallel (~> 1.10) + parser (>= 3.2.0.0) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.28.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.30.0) + parser (>= 3.2.1.0) + rubocop-capybara (2.19.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.24.0) + rubocop (~> 1.33) + rubocop-graphql (0.19.0) + rubocop (>= 0.87, < 2) + rubocop-performance (1.19.1) + rubocop (>= 1.7.0, < 2.0) + rubocop-ast (>= 0.4.0) + rubocop-rails (2.22.2) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.33.0, < 2.0) + rubocop-ast (>= 1.30.0, < 2.0) + rubocop-rspec (2.25.0) + rubocop (~> 1.40) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + ruby-progressbar (1.13.0) + safe_yaml (1.0.4) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.5.0) + webmock (3.19.1) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) + +PLATFORMS + arm64-darwin-22 + +DEPENDENCIES + gitlab-housekeeper! + gitlab-rspec! + gitlab-styles + pry + rake (~> 13.0) + rspec (~> 3.0) + rspec-rails + rubocop-rspec + webmock + +BUNDLED WITH + 2.4.21 diff --git a/gems/gitlab-housekeeper/README.md b/gems/gitlab-housekeeper/README.md new file mode 100644 index 00000000000..f707b99c6f0 --- /dev/null +++ b/gems/gitlab-housekeeper/README.md @@ -0,0 +1,34 @@ +# Gitlab::Housekeeper + +Housekeeping following https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134487 + +## Running + +Technically you can skip steps 1-2 below if you don't want to create a fork but +it's recommended as using a bot account with no permissions in +`gitlab-org/gitlab` will ensure we can't cause much damage if the script makes +a mistake. The alternative of using your own API token with it's permissions to +`gitlab-org/gitlab` has slightly more risks. + +1. Create a fork of `gitlab-org/gitlab` where your MRs will come from +1. Create a project access token for that project +1. Set `housekeeper` remote to the fork you created + ``` + git remote add housekeeper git@gitlab.com:DylanGriffith/gitlab.git + ``` +1. Open a Postgres.ai tunnel on localhost port 6305 +1. Set the Postgres AI env vars matching the tunnel details for your tunnel + ``` + export POSTGRES_AI_CONNECTION_STRING='host=localhost port=6305 user=dylan dbname=gitlabhq_dblab' + export POSTGRES_AI_PASSWORD='the-password' + ``` +1. Set the GitLab client details. Will be used to create MR from housekeeper remote: + ``` + export HOUSEKEEPER_FORK_PROJECT_ID=52263761 # Same project as housekeeper remote + export HOUSEKEEPER_TARGET_PROJECT_ID=52263761 # Can be 278964 (gitlab-org/gitlab) when ready to create real MRs + export HOUSEKEEPER_GITLAB_API_TOKEN=the-api-token + ``` +1. Run it: + ``` + bundle exec gitlab-housekeeper -d -m3 -r keeps/overdue_finalize_background_migration.rb -k Keeps::OverdueFinalizeBackgroundMigration + ``` diff --git a/gems/gitlab-housekeeper/Rakefile b/gems/gitlab-housekeeper/Rakefile new file mode 100644 index 00000000000..b6ae734104e --- /dev/null +++ b/gems/gitlab-housekeeper/Rakefile @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require "bundler/gem_tasks" +require "rspec/core/rake_task" + +RSpec::Core::RakeTask.new(:spec) + +task default: :spec diff --git a/gems/gitlab-housekeeper/bin/gitlab-housekeeper b/gems/gitlab-housekeeper/bin/gitlab-housekeeper new file mode 100755 index 00000000000..75c547f81ff --- /dev/null +++ b/gems/gitlab-housekeeper/bin/gitlab-housekeeper @@ -0,0 +1,32 @@ +#!/usr/bin/env ruby + +require "optparse" +require 'gitlab/housekeeper' + +options = {} + +OptionParser.new do |opts| + opts.banner = 'Creates merge requests that can be inferred from the current state of the codebase' + + opts.on('-m=M', '--max-mrs=M', Integer, 'Limit of MRs to create. Defaults to 1.') do |m| + options[:max_mrs] = m + end + + opts.on('-d', '--dry-run', 'Dry-run only. Print the MR titles, descriptions and diffs') do + options[:dry_run] = true + end + + opts.on('-r lib/foo.rb lib/bar.rb', '--require lib/foo.rb lib/bar.rb', Array, 'Require keeps specified') do |r| + options[:require] = r + end + + opts.on('-k OverdueFinalizeBackgroundMigration,AnotherKeep', '--keeps OverdueFinalizeBackgroundMigration,AnotherKeep', Array, 'Require keeps specified') do |k| + options[:keeps] = k + end + + opts.on('-h', '--help', 'Prints this help') do + abort opts.to_s + end +end.parse! + +Gitlab::Housekeeper::Runner.new(**options).run diff --git a/gems/gitlab-housekeeper/gitlab-housekeeper.gemspec b/gems/gitlab-housekeeper/gitlab-housekeeper.gemspec new file mode 100644 index 00000000000..798ca5dcfe6 --- /dev/null +++ b/gems/gitlab-housekeeper/gitlab-housekeeper.gemspec @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require_relative 'lib/gitlab/housekeeper/version' + +Gem::Specification.new do |spec| + spec.name = "gitlab-housekeeper" + spec.version = Gitlab::Housekeeper::VERSION + spec.authors = ["group::tenant-scale"] + spec.email = ["engineering@gitlab.com"] + + spec.summary = "Gem summary" + spec.description = "Housekeeping following https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134487" + spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper" + spec.license = "MIT" + spec.required_ruby_version = ">= 3.0" + spec.metadata["rubygems_mfa_required"] = "true" + + spec.files = Dir['lib/**/*.rb'] + spec.require_paths = ["lib"] + spec.executables = ['gitlab-housekeeper'] + + spec.add_runtime_dependency 'httparty' + spec.add_runtime_dependency 'rubocop' + + spec.add_development_dependency 'gitlab-styles' + spec.add_development_dependency 'rspec-rails' + spec.add_development_dependency "rubocop-rspec" + spec.add_development_dependency 'webmock' +end diff --git a/gems/gitlab-housekeeper/lib/gitlab/housekeeper.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper.rb new file mode 100644 index 00000000000..e9f97e629f4 --- /dev/null +++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require "gitlab/housekeeper/version" +require "gitlab/housekeeper/runner" + +module Gitlab + module Housekeeper + Error = Class.new(StandardError) + # Your code goes here... + end +end diff --git a/gems/gitlab-housekeeper/lib/gitlab/housekeeper/git.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/git.rb new file mode 100644 index 00000000000..94bddbaf95f --- /dev/null +++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/git.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +require 'logger' +require 'gitlab/housekeeper/shell' + +module Gitlab + module Housekeeper + class Git + def initialize(logger:, branch_from: 'master') + @logger = logger + @branch_from = branch_from + end + + def commit_in_branch(change) + branch_name = branch_name(change.identifiers) + + create_commit(branch_name, change) + + branch_name + end + + def with_branch_from_branch + stashed = false + current_branch = Shell.execute('git', 'branch', '--show-current').chomp + + result = Shell.execute('git', 'stash') + stashed = !result.include?('No local changes to save') + + Shell.execute("git", "checkout", @branch_from) + + yield + ensure + Shell.execute("git", "checkout", current_branch) + Shell.execute('git', 'stash', 'pop') if stashed + end + + def create_commit(branch_name, change) + current_branch = Shell.execute('git', 'branch', '--show-current').chomp + + begin + Shell.execute("git", "branch", '-D', branch_name) + rescue Shell::Error # Might not exist yet + end + + Shell.execute("git", "checkout", "-b", branch_name) + Shell.execute("git", "add", *change.changed_files) + + commit_message = <<~MSG + #{change.title} + + #{change.description} + + This commit was generated by + [gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139492). + + Changelog: other + MSG + + Shell.execute("git", "commit", "-m", commit_message) + + ensure + Shell.execute("git", "checkout", current_branch) + end + + def branch_name(identifiers) + # Hyphen-case each identifier then join together with hyphens. + branch_name = identifiers + .map { |i| i.gsub(/[[:upper:]]/) { |w| "-#{w.downcase}" } } + .join('-') + .delete_prefix("-") + + # Truncate if it's too long and add a digest + if branch_name.length > 240 + branch_name = branch_name[0...200] + OpenSSL::Digest::SHA256.hexdigest(branch_name)[0...15] + end + + branch_name + end + end + end +end diff --git a/gems/gitlab-housekeeper/lib/gitlab/housekeeper/gitlab_client.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/gitlab_client.rb new file mode 100644 index 00000000000..b28d44195cb --- /dev/null +++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/gitlab_client.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +require 'httparty' +require 'json' + +module Gitlab + module Housekeeper + class GitlabClient + Error = Class.new(StandardError) + + def initialize + @token = ENV.fetch("HOUSEKEEPER_GITLAB_API_TOKEN") + @base_uri = 'https://gitlab.com/api/v4' + end + + def create_or_update_merge_request( + source_project_id:, + title:, + description:, + source_branch:, + target_branch:, + target_project_id: + ) + existing_iid = get_existing_merge_request( + source_project_id: source_project_id, + source_branch: source_branch, + target_branch: target_branch, + target_project_id: target_project_id + ) + + if existing_iid + update_existing_merge_request( + existing_iid: existing_iid, + title: title, + description: description, + target_project_id: target_project_id + ) + else + create_merge_request( + source_project_id: source_project_id, + title: title, + description: description, + source_branch: source_branch, + target_branch: target_branch, + target_project_id: target_project_id + ) + end + end + + private + + def get_existing_merge_request(source_project_id:, source_branch:, target_branch:, target_project_id:) + response = HTTParty.get("#{@base_uri}/projects/#{target_project_id}/merge_requests", + query: { + state: :opened, + source_branch: source_branch, + target_branch: target_branch, + source_project_id: source_project_id + }, + headers: { + 'Private-Token' => @token + }) + + unless (200..299).cover?(response.code) + raise Error, + "Failed with response code: #{response.code} and body:\n#{response.body}" + end + + data = JSON.parse(response.body) + + return nil if data.empty? + + iids = data.pluck('iid') + + raise Error, "More than one matching MR exists: iids: #{iids.join(',')}" unless data.size == 1 + + iids.first + end + + def create_merge_request( + source_project_id:, title:, description:, source_branch:, target_branch:, + target_project_id:) + response = HTTParty.post("#{@base_uri}/projects/#{source_project_id}/merge_requests", body: { + title: title, + description: description, + source_branch: source_branch, + target_branch: target_branch, + target_project_id: target_project_id + }.to_json, + headers: { + 'Private-Token' => @token, + 'Content-Type' => 'application/json' + }) + + return if (200..299).cover?(response.code) + + raise Error, + "Failed with response code: #{response.code} and body:\n#{response.body}" + end + + def update_existing_merge_request(existing_iid:, title:, description:, target_project_id:) + response = HTTParty.put("#{@base_uri}/projects/#{target_project_id}/merge_requests/#{existing_iid}", body: { + title: title, + description: description + }.to_json, + headers: { + 'Private-Token' => @token, + 'Content-Type' => 'application/json' + }) + + return if (200..299).cover?(response.code) + + raise Error, + "Failed with response code: #{response.code} and body:\n#{response.body}" + end + end + end +end diff --git a/gems/gitlab-housekeeper/lib/gitlab/housekeeper/keep.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/keep.rb new file mode 100644 index 00000000000..06d4c6d1afe --- /dev/null +++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/keep.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab + module Housekeeper + class Keep # rubocop:disable Lint/EmptyClass + end + end +end diff --git a/gems/gitlab-housekeeper/lib/gitlab/housekeeper/runner.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/runner.rb new file mode 100644 index 00000000000..98e28d3de97 --- /dev/null +++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/runner.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require 'gitlab/housekeeper/keep' +require "gitlab/housekeeper/gitlab_client" +require "gitlab/housekeeper/git" +require 'digest' + +module Gitlab + module Housekeeper + Change = Struct.new(:identifiers, :title, :description, :changed_files) + + class Runner + def initialize(max_mrs: 1, dry_run: false, require: [], keeps: nil) + @max_mrs = max_mrs + @dry_run = dry_run + @logger = Logger.new($stdout) + require_keeps(require) + + @keeps = if keeps + keeps.map { |k| k.is_a?(String) ? k.constantize : k } + else + all_keeps + end + end + + def run + created = 0 + + git.with_branch_from_branch do + @keeps.each do |keep| + keep.new.each do |change| + branch_name = git.commit_in_branch(change) + + if @dry_run + dry_run(change, branch_name) + else + create(change, branch_name) + end + + created += 1 + break if created >= @max_mrs + end + break if created >= @max_mrs + end + end + + puts "Housekeeper created #{created} MRs" + end + + def git + @git ||= ::Gitlab::Housekeeper::Git.new(logger: @logger) + end + + def require_keeps(files) + files.each do |r| + require(Pathname(r).expand_path.to_s) + end + end + + def dry_run(change, branch_name) + puts + puts "# #{change.title}" + puts + puts change.description + puts + puts Shell.execute('git', '--no-pager', 'diff', 'master', branch_name, '--', *change.changed_files) + end + + def create(change, branch_name) + dry_run(change, branch_name) + + Shell.execute('git', 'push', '-f', 'housekeeper', "#{branch_name}:#{branch_name}") + + gitlab_client.create_or_update_merge_request( + source_project_id: housekeeper_fork_project_id, + title: change.title, + description: change.description, + source_branch: branch_name, + target_branch: 'master', + target_project_id: housekeeper_target_project_id + ) + end + + def housekeeper_fork_project_id + ENV.fetch('HOUSEKEEPER_FORK_PROJECT_ID') + end + + def housekeeper_target_project_id + ENV.fetch('HOUSEKEEPER_TARGET_PROJECT_ID') + end + + def gitlab_client + @gitlab_client ||= GitlabClient.new + end + + def all_keeps + @all_keeps ||= ObjectSpace.each_object(Class).select { |klass| klass < Keep } + end + end + end +end diff --git a/gems/gitlab-housekeeper/lib/gitlab/housekeeper/shell.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/shell.rb new file mode 100644 index 00000000000..ed51073f0ed --- /dev/null +++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/shell.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'open3' + +module Gitlab + module Housekeeper + class Shell + Error = Class.new(StandardError) + + def self.execute(*cmd) + stdin, stdout, stderr, wait_thr = Open3.popen3(*cmd) + + stdin.close + out = stdout.read + stdout.close + err = stderr.read + stderr.close + + exit_status = wait_thr.value + + raise Error, "Failed with #{exit_status}\n#{out}\n#{err}\n" unless exit_status.success? + + out + err + end + end + end +end diff --git a/gems/gitlab-housekeeper/lib/gitlab/housekeeper/version.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/version.rb new file mode 100644 index 00000000000..da4cce3f5b3 --- /dev/null +++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/version.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Gitlab + module Housekeeper + VERSION = "0.1.0" + end +end diff --git a/gems/gitlab-housekeeper/spec/gitlab/housekeeper/git_spec.rb b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/git_spec.rb new file mode 100644 index 00000000000..23aba5fc791 --- /dev/null +++ b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/git_spec.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'fileutils' +require 'tmpdir' + +# rubocop:disable RSpec/MultipleMemoizedHelpers +RSpec.describe ::Gitlab::Housekeeper::Git do + let(:logger) { instance_double(Logger, info: nil) } + let(:git) { described_class.new(logger: logger) } + let(:repository_path) { Pathname(Dir.mktmpdir) } + let(:test_branch_name) { 'gitlab-housekeeper--test-branch' } + let(:file_in_master) { 'file_in_master.txt' } + let(:file_in_another_branch) { 'file_in_another_branch.txt' } + + def setup_master_branch + File.write(file_in_master, 'File already in master!') + + ::Gitlab::Housekeeper::Shell.execute('git', 'init') + ::Gitlab::Housekeeper::Shell.execute('git', 'checkout', '-b', 'master') + ::Gitlab::Housekeeper::Shell.execute('git', 'add', file_in_master) + ::Gitlab::Housekeeper::Shell.execute('git', 'commit', '-m', 'Initial commit!') + end + + def setup_and_checkout_another_branch + ::Gitlab::Housekeeper::Shell.execute('git', 'checkout', '-b', 'another-branch') + + File.write(file_in_another_branch, 'File in another unrelated branch should not be in new branch!') + ::Gitlab::Housekeeper::Shell.execute('git', 'add', file_in_another_branch) + ::Gitlab::Housekeeper::Shell.execute('git', 'commit', '-m', 'Commit in unrelated branch should not be included') + end + + before do + @previous_dir = Dir.pwd + Dir.chdir(repository_path) + + # Make sure there is a master branch with something to branch from + setup_master_branch + setup_and_checkout_another_branch + end + + after do + Dir.chdir(@previous_dir) if @previous_dir # rubocop:disable RSpec/InstanceVariable + FileUtils.rm_rf(repository_path) + end + + describe '#with_branch_from_branch and #commit_in_branch' do + let(:file_not_to_commit) { repository_path.join('test_file_not_to_commit.txt') } + let(:test_file1) { 'test_file1.txt' } + let(:test_file2) { 'files/test_file2.txt' } + + it 'commits the given change details to the given branch name' do + title = "The commit title" + description = <<~COMMIT + The commit description can be + split over multiple lines! + COMMIT + + identifiers = %w[GitlabHousekeeper TestBranch] + + Dir.mkdir('files') + File.write(test_file1, "Content in file 1!") + File.write(test_file2, "Other content in file 2!") + File.write(file_not_to_commit, 'Do not commit!') + + changed_files = [test_file1, test_file2] + + change = ::Gitlab::Housekeeper::Change.new( + identifiers, + title, + description, + changed_files + ) + + branch_name = nil + git.with_branch_from_branch do + branch_name = git.commit_in_branch(change) + end + + expect(branch_name).to eq(test_branch_name) + + branches = ::Gitlab::Housekeeper::Shell.execute('git', 'branch') + expect(branches).to include(branch_name) + + current_commit_on_another_branch = ::Gitlab::Housekeeper::Shell.execute('git', 'show') + expect(current_commit_on_another_branch).to include('Commit in unrelated branch should not be included') + + expected = <<~COMMIT + The commit title + + The commit description can be + split over multiple lines! + + + This commit was generated by + [gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139492). + + Changelog: other + + diff --git a/files/test_file2.txt b/files/test_file2.txt + new file mode 100644 + index 0000000..ff205e0 + --- /dev/null + +++ b/files/test_file2.txt + @@ -0,0 +1 @@ + +Other content in file 2! + \\ No newline at end of file + diff --git a/test_file1.txt b/test_file1.txt + new file mode 100644 + index 0000000..8dd3371 + --- /dev/null + +++ b/test_file1.txt + @@ -0,0 +1 @@ + +Content in file 1! + \\ No newline at end of file + COMMIT + + commit = ::Gitlab::Housekeeper::Shell.execute('git', 'show', branch_name).gsub(/\s/, '') + expected_without_whitespace = expected.gsub(/\s/, '') + expect(commit).to include(expected_without_whitespace) + + ::Gitlab::Housekeeper::Shell.execute('git', 'checkout', branch_name) + expect(File).to exist(file_in_master) + expect(File).not_to exist(file_in_another_branch) + end + end +end +# rubocop:enable RSpec/MultipleMemoizedHelpers diff --git a/gems/gitlab-housekeeper/spec/gitlab/housekeeper/gitlab_client_spec.rb b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/gitlab_client_spec.rb new file mode 100644 index 00000000000..36b2afdc306 --- /dev/null +++ b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/gitlab_client_spec.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'gitlab/housekeeper/gitlab_client' + +RSpec.describe ::Gitlab::Housekeeper::GitlabClient do + let(:client) { described_class.new } + + describe '#create_or_update_merge_request' do + let(:params) do + { + source_project_id: 123, + title: 'A new merge request!', + description: 'This merge request is pretty good.', + source_branch: 'the-source-branch', + target_branch: 'the-target-branch', + target_project_id: 456 + } + end + + let(:existing_mrs) { [] } + + before do + stub_env('HOUSEKEEPER_GITLAB_API_TOKEN', 'the-api-token') + + # Stub the check to see if the merge request already exists + stub_request(:get, "https://gitlab.com/api/v4/projects/456/merge_requests?state=opened&source_branch=the-source-branch&target_branch=the-target-branch&source_project_id=123") + .with( + headers: { + 'Private-Token' => 'the-api-token' + } + ) + .to_return(status: 200, body: existing_mrs.to_json) + end + + it 'calls the GitLab API passing the token' do + stub = stub_request(:post, "https://gitlab.com/api/v4/projects/123/merge_requests") + .with( + body: { + title: "A new merge request!", + description: "This merge request is pretty good.", + source_branch: "the-source-branch", + target_branch: "the-target-branch", + target_project_id: 456 + }, + headers: { + 'Content-Type' => 'application/json', + 'Private-Token' => 'the-api-token' + }) + .to_return(status: 200, body: "") + + client.create_or_update_merge_request(**params) + + expect(stub).to have_been_requested + end + + context 'when the merge request for the branch already exists' do + let(:existing_mrs) do + [{ iid: 1234 }] + end + + it 'updates the merge request' do + stub = stub_request(:put, "https://gitlab.com/api/v4/projects/456/merge_requests/1234") + .with( + body: { + title: "A new merge request!", + description: "This merge request is pretty good." + }.to_json, + headers: { + 'Content-Type' => 'application/json', + 'Private-Token' => 'the-api-token' + }) + .to_return(status: 200, body: "") + + client.create_or_update_merge_request(**params) + expect(stub).to have_been_requested + end + + context 'when multiple merge requests exist' do + let(:existing_mrs) do + [{ iid: 1234 }, { iid: 5678 }] + end + + it 'raises since we do not expect this to be possible' do + expect { client.create_or_update_merge_request(**params) }.to raise_error(described_class::Error) + end + end + end + + it 'raises an error when unsuccessful response' do + stub_request(:post, "https://gitlab.com/api/v4/projects/123/merge_requests") + .to_return(status: 400, body: "Real bad error") + + expect do + client.create_or_update_merge_request(**params) + end.to raise_error(described_class::Error, a_string_matching('Real bad error')) + end + end +end diff --git a/gems/gitlab-housekeeper/spec/gitlab/housekeeper/runner_spec.rb b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/runner_spec.rb new file mode 100644 index 00000000000..be9ced21fba --- /dev/null +++ b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/runner_spec.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'gitlab/housekeeper/runner' + +RSpec.describe ::Gitlab::Housekeeper::Runner do + let(:fake_keep) { double(:fake_keep) } # rubocop:disable RSpec/VerifiedDoubles + + let(:change1) do + ::Gitlab::Housekeeper::Change.new( + %w[the identifier for the first change], + "The title of MR1", + "The description of the MR", + ['change1.txt', 'change2.txt'] + ) + end + + let(:change2) do + ::Gitlab::Housekeeper::Change.new( + %w[the identifier for the second change], + "The title of MR2", + "The description of the MR", + ['change1.txt', 'change2.txt'] + ) + end + + let(:change3) do + ::Gitlab::Housekeeper::Change.new( + %w[the identifier for the third change], + "The title of MR3", + "The description of the MR", + ['change1.txt', 'change2.txt'] + ) + end + + before do + fake_keep_instance = double(:fake_keep_instance) # rubocop:disable RSpec/VerifiedDoubles + allow(fake_keep).to receive(:new).and_return(fake_keep_instance) + + allow(fake_keep_instance).to receive(:each) + .and_yield(change1) + .and_yield(change2) + .and_yield(change3) + end + + describe '#run' do + before do + stub_env('HOUSEKEEPER_FORK_PROJECT_ID', '123') + stub_env('HOUSEKEEPER_TARGET_PROJECT_ID', '456') + end + + it 'loops over the keeps and creates MRs limited by max_mrs' do + # Branches get created + git = instance_double(::Gitlab::Housekeeper::Git) + expect(::Gitlab::Housekeeper::Git).to receive(:new) + .and_return(git) + expect(git).to receive(:with_branch_from_branch) + .and_yield + expect(git).to receive(:commit_in_branch).with(change1) + .and_return('the-identifier-for-the-first-change') + expect(git).to receive(:commit_in_branch).with(change2) + .and_return('the-identifier-for-the-second-change') + + # Branches get shown and pushed + expect(::Gitlab::Housekeeper::Shell).to receive(:execute) + .with('git', '--no-pager', 'diff', 'master', + 'the-identifier-for-the-first-change', '--', 'change1.txt', 'change2.txt') + expect(::Gitlab::Housekeeper::Shell).to receive(:execute) + .with('git', 'push', '-f', 'housekeeper', + 'the-identifier-for-the-first-change:the-identifier-for-the-first-change') + expect(::Gitlab::Housekeeper::Shell).to receive(:execute) + .with('git', '--no-pager', 'diff', 'master', + 'the-identifier-for-the-second-change', '--', 'change1.txt', 'change2.txt') + expect(::Gitlab::Housekeeper::Shell).to receive(:execute) + .with('git', 'push', '-f', 'housekeeper', + 'the-identifier-for-the-second-change:the-identifier-for-the-second-change') + + # Merge requests get created + gitlab_client = instance_double(::Gitlab::Housekeeper::GitlabClient) + expect(::Gitlab::Housekeeper::GitlabClient).to receive(:new) + .and_return(gitlab_client) + expect(gitlab_client).to receive(:create_or_update_merge_request) + .with( + source_project_id: '123', + title: 'The title of MR1', + description: 'The description of the MR', + source_branch: 'the-identifier-for-the-first-change', + target_branch: 'master', + target_project_id: '456' + ) + expect(gitlab_client).to receive(:create_or_update_merge_request) + .with( + source_project_id: '123', + title: 'The title of MR2', + description: 'The description of the MR', + source_branch: 'the-identifier-for-the-second-change', + target_branch: 'master', + target_project_id: '456' + ) + + described_class.new(max_mrs: 2, keeps: [fake_keep]).run + end + end +end diff --git a/gems/gitlab-housekeeper/spec/gitlab/housekeeper/shell_spec.rb b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/shell_spec.rb new file mode 100644 index 00000000000..2c7b3fb01c8 --- /dev/null +++ b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/shell_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'gitlab/housekeeper/shell' + +RSpec.describe ::Gitlab::Housekeeper::Shell do + describe '.execute' do + it 'delegates to popen3 and returns stdout' do + expect(Open3).to receive(:popen3).with('echo', 'hello world') + .and_call_original + + expect(described_class.execute('echo', 'hello world')).to eq("hello world\n") + end + + it 'raises when result is not successful' do + expect do + described_class.execute('cat', 'definitelynotafile') + end.to raise_error( + described_class::Error, + a_string_matching("cat: definitelynotafile: No such file or directory\n") + ) + end + end +end diff --git a/gems/gitlab-housekeeper/spec/gitlab/housekeeper_spec.rb b/gems/gitlab-housekeeper/spec/gitlab/housekeeper_spec.rb new file mode 100644 index 00000000000..798ba7f9386 --- /dev/null +++ b/gems/gitlab-housekeeper/spec/gitlab/housekeeper_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +RSpec.describe Gitlab::Housekeeper do + it "has a version number" do + expect(Gitlab::Housekeeper::VERSION).not_to be nil + end +end diff --git a/gems/gitlab-housekeeper/spec/spec_helper.rb b/gems/gitlab-housekeeper/spec/spec_helper.rb new file mode 100644 index 00000000000..f2ebb2298bd --- /dev/null +++ b/gems/gitlab-housekeeper/spec/spec_helper.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'rspec/mocks' +require "gitlab/housekeeper" +require "gitlab/housekeeper/git" +require 'webmock/rspec' +require 'gitlab/rspec/all' + +RSpec.configure do |config| + config.include StubENV + # Enable flags like --only-failures and --next-failure + config.example_status_persistence_file_path = ".rspec_status" + + # Disable RSpec exposing methods globally on `Module` and `main` + config.disable_monkey_patching! + + config.expect_with :rspec do |c| + c.syntax = :expect + end +end diff --git a/keeps/helpers/postgres_ai.rb b/keeps/helpers/postgres_ai.rb new file mode 100644 index 00000000000..c4cee13325c --- /dev/null +++ b/keeps/helpers/postgres_ai.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Keeps + module Helpers + class PostgresAi + Error = Class.new(StandardError) + + def initialize + raise Error, "No credentials supplied" unless connection_string.present? && password.present? + end + + def fetch_background_migration_status(job_class_name) + query = <<~SQL + SELECT id, created_at, updated_at, finished_at, started_at, status, job_class_name + FROM batched_background_migrations + WHERE job_class_name = $1::text + SQL + + pg_client.exec_params(query, [job_class_name]) + end + + private + + def connection_string + ENV["POSTGRES_AI_CONNECTION_STRING"] + end + + def password + ENV["POSTGRES_AI_PASSWORD"] + end + + def pg_client + @pg_client ||= PG.connect(connection_string, password: password) + end + end + end +end diff --git a/keeps/overdue_finalize_background_migration.rb b/keeps/overdue_finalize_background_migration.rb new file mode 100644 index 00000000000..ebeca23a17c --- /dev/null +++ b/keeps/overdue_finalize_background_migration.rb @@ -0,0 +1,205 @@ +# frozen_string_literal: true + +require ::File.expand_path('../config/environment', __dir__) +require_relative '../lib/generators/post_deployment_migration/post_deployment_migration_generator' +require_relative './helpers/postgres_ai' + +module Keeps + class OverdueFinalizeBackgroundMigration < ::Gitlab::Housekeeper::Keep + CUTOFF_MILESTONE = '16.4' + + def initialize; end + + def each + each_batched_background_migration do |migration_yaml_file, migration| + next unless before_cuttoff_milestone?(migration['milestone']) + + job_name = migration['migration_job_name'] + + next if migration_finalized?(job_name) + + migration_record = fetch_migration_status(job_name) + + next unless migration_record + + # Finalize the migration + title = "Finalize migration #{job_name}" + + identifiers = [self.class.name.demodulize, job_name] + + last_migration_file = last_migration_for_job(job_name) + + # rubocop:disable Gitlab/DocUrl -- Not running inside rails application + description = <<~MARKDOWN + This migration was finished at `#{migration_record.finished_at || migration_record.updated_at}`, you can confirm + the status using our + [batched background migration chatops commands](https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#monitor-the-progress-and-status-of-a-batched-background-migration). + To confirm it is finished you can run: + + ``` + /chatops run batched_background_migrations status #{migration_record.id} + ``` + + The last time this background migration was triggered was in [#{last_migration_file}](https://gitlab.com/gitlab-org/gitlab/-/blob/master/#{last_migration_file}) + + You can read more about the process for finalizing batched background migrations in + https://docs.gitlab.com/ee/development/database/batched_background_migrations.html . + + As part of our process we want to ensure all batched background migrations have had at least one + [required stop](https://docs.gitlab.com/ee/development/database/required_stops.html) + to process the migration. Therefore we can finalize any batched background migration that was added before the + last required stop. + + This merge request was created using the + [gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139492) + gem. + MARKDOWN + # rubocop:enable Gitlab/DocUrl + + queue_method_node = find_queue_method_node(last_migration_file) + + # TODO: Can runner figure out what changed during this block? + migration_name = truncate_migration_name("Finalize#{migration['migration_job_name']}") + PostDeploymentMigration::PostDeploymentMigrationGenerator + .source_root('generator_templates/post_deployment_migration/post_deployment_migration/') + generator = ::PostDeploymentMigration::PostDeploymentMigrationGenerator.new([migration_name], force: true) + migration_file = generator.invoke_all.first + changed_files = [migration_file] + + add_ensure_call_to_migration(migration_file, queue_method_node, job_name) + ::Gitlab::Housekeeper::Shell.execute('rubocop', '-a', migration_file) + + digest = Digest::SHA256.hexdigest(generator.migration_number) + digest_file = Pathname.new('db').join('schema_migrations', generator.migration_number.to_s).to_s + File.open(digest_file, 'w') { |f| f.write(digest) } + + add_finalized_by_to_yaml(migration_yaml_file, generator.migration_number) + + changed_files << digest_file + changed_files << migration_yaml_file + + to_create = ::Gitlab::Housekeeper::Change.new(identifiers, title, description, changed_files) + yield(to_create) + end + end + + def truncate_migration_name(migration_name) + # File names not allowed to exceed 100 chars due to Cop/FilenameLength so we truncate to 70 because there will be + # underscores added. + + migration_name[0...70] + end + + def add_finalized_by_to_yaml(yaml_file, migration_number) + content = YAML.load_file(yaml_file) + content['finalized_by'] = migration_number + File.open(yaml_file, 'w') { |f| f.write(YAML.dump(content)) } + end + + def last_migration_for_job(job_name) + result = ::Gitlab::Housekeeper::Shell.execute('git', 'grep', '--name-only', "MIGRATION = .#{job_name}.").chomp + result = result.each_line.select do |file| + File.read(file).include?('queue_batched_background_migration') + end.max + + raise "Could not find migration for #{job_name}" unless result.present? + + result + end + + def add_ensure_call_to_migration(file, queue_method_node, job_name) + source = RuboCop::ProcessedSource.new(File.read(file), 3.1) + ast = source.ast + source_buffer = source.buffer + rewriter = Parser::Source::TreeRewriter.new(source_buffer) + + up_method = ast.children[2].each_child_node(:def).find do |child| + child.method_name == :up + end + + table_name = queue_method_node.children[3] + column_name = queue_method_node.children[4] + job_arguments = queue_method_node.children[5..].select { |s| s.type != :hash } # All remaining non-keyword args + + gitlab_schema = ::Gitlab::Database::GitlabSchema.table_schema(table_name.value.to_s) + + added_content = <<~RUBY.strip + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :#{gitlab_schema} + + def up + ensure_batched_background_migration_is_finished( + job_class_name: '#{job_name}', + table_name: #{table_name.source}, + column_name: #{column_name.source}, + job_arguments: [#{job_arguments.map(&:source).join(', ')}], + finalize: true + ) + end + RUBY + + rewriter.replace(up_method.loc.expression, added_content) + + content = strip_comments(rewriter.process) + + File.write(file, content) + end + + def strip_comments(code) + result = [] + code.each_line.with_index do |line, index| + result << line unless index > 0 && line.lstrip.start_with?('#') + end + result.join + end + + def fetch_migration_status(job_name) + result = postgres_ai.fetch_background_migration_status(job_name) + + return unless result.count == 1 + + migration_model = ::Gitlab::Database::BackgroundMigration::BatchedMigration.new(result.first) + + migration_model if migration_model.finished? + end + + def postgres_ai + @postgres_ai ||= Keeps::Helpers::PostgresAi.new + end + + def migration_finalized?(job_name) + result = `git grep --name-only "#{job_name}"`.chomp + result.each_line.select do |file| + File.read(file.chomp).include?('ensure_batched_background_migration_is_finished') + end.any? + end + + def find_queue_method_node(file) + source = RuboCop::ProcessedSource.new(File.read(file), 3.1) + ast = source.ast + + up_method = ast.children[2].children.find do |child| + child.def_type? && child.method_name == :up + end + + up_method.each_descendant.find do |child| + child && child.send_type? && child.method_name == :queue_batched_background_migration + end + end + + def before_cuttoff_milestone?(milestone) + Gem::Version.new(milestone) < Gem::Version.new(CUTOFF_MILESTONE) + end + + def each_batched_background_migration + all_batched_background_migration_files.map do |f| + yield(f, YAML.load_file(f)) + end + end + + def all_batched_background_migration_files + Dir.glob("db/docs/batched_background_migrations/*.yml") + end + end +end diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index 85d137abd50..c334f618847 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -126,38 +126,7 @@ module API def self.integrations { - 'apple-app-store' => [ - { - required: true, - name: :app_store_issuer_id, - type: String, - desc: 'The Apple App Store Connect Issuer ID' - }, - { - required: true, - name: :app_store_key_id, - type: String, - desc: 'The Apple App Store Connect Key ID' - }, - { - required: true, - name: :app_store_private_key, - type: String, - desc: 'The Apple App Store Connect Private Key' - }, - { - required: true, - name: :app_store_private_key_file_name, - type: String, - desc: 'The Apple App Store Connect Private Key File Name' - }, - { - required: false, - name: :app_store_protected_refs, - type: ::Grape::API::Boolean, - desc: 'Only enable for protected refs' - } - ], + 'apple-app-store' => ::Integrations::AppleAppStore.api_fields, 'asana' => [ { required: true, diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 578cfb52714..8e894be4fc4 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -436,8 +436,7 @@ module Gitlab end def unavailable_scopes_for_resource(resource) - unavailable_observability_scopes_for_resource(resource) + - unavailable_ai_features_scopes_for_resource(resource) + unavailable_observability_scopes_for_resource(resource) end def unavailable_observability_scopes_for_resource(resource) @@ -447,10 +446,6 @@ module Gitlab OBSERVABILITY_SCOPES end - def unavailable_ai_features_scopes_for_resource(_resource) - AI_FEATURES_SCOPES - end - def non_admin_available_scopes API_SCOPES + REPOSITORY_SCOPES + registry_scopes + OBSERVABILITY_SCOPES + AI_FEATURES_SCOPES end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 2d958691959..f5478c84e57 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -4430,9 +4430,6 @@ msgstr "" msgid "After it is removed, the fork relationship can only be restored by using the API. This project will no longer be able to receive or send merge requests to the upstream project or other forks." msgstr "" -msgid "After the Apple App Store Connect integration is activated, the following protected variables will be created for CI/CD use." -msgstr "" - msgid "After the export is complete, download the data file from a notification email or from this page. You can then import the data file from the %{strong_text_start}Create new group%{strong_text_end} page of another GitLab instance." msgstr "" @@ -4442,6 +4439,9 @@ msgstr "" msgid "After you enable the integration, the following protected variable is created for CI/CD use:" msgstr "" +msgid "After you enable the integration, the following protected variables are created for CI/CD use:" +msgstr "" + msgid "After you've reviewed these contribution guidelines, you'll be all set to" msgstr "" @@ -5856,6 +5856,27 @@ msgstr "" msgid "Append the comment with %{tableflip}" msgstr "" +msgid "Append the hostname of your GitLab instance to the status check name." +msgstr "" + +msgid "Apple App Store Connect private key file name." +msgstr "" + +msgid "Apple App Store Connect private key." +msgstr "" + +msgid "AppleAppStore|Apple App Store Connect issuer ID" +msgstr "" + +msgid "AppleAppStore|Apple App Store Connect issuer ID." +msgstr "" + +msgid "AppleAppStore|Apple App Store Connect key ID" +msgstr "" + +msgid "AppleAppStore|Apple App Store Connect key ID." +msgstr "" + msgid "AppleAppStore|Drag your Private Key file here or %{linkStart}click to upload%{linkEnd}." msgstr "" @@ -5868,16 +5889,10 @@ msgstr "" msgid "AppleAppStore|Leave empty to use your current Private Key." msgstr "" -msgid "AppleAppStore|Only set variables on protected branches and tags" -msgstr "" - msgid "AppleAppStore|Protected branches and tags only" msgstr "" -msgid "AppleAppStore|The Apple App Store Connect Issuer ID." -msgstr "" - -msgid "AppleAppStore|The Apple App Store Connect Key ID." +msgid "AppleAppStore|Set variables on protected branches and tags only." msgstr "" msgid "AppleAppStore|The Apple App Store Connect Private Key (.p8)" @@ -22054,9 +22069,15 @@ msgstr "" msgid "GitHub API rate limit exceeded. Try again after %{reset_time}" msgstr "" +msgid "GitHub API token with `repo:status` OAuth scope." +msgstr "" + msgid "GitHub import" msgstr "" +msgid "GitHub repository URL." +msgstr "" + msgid "GitHubImporter|*Merged by: %{author} at %{timestamp}*" msgstr "" @@ -52558,9 +52579,6 @@ msgstr "" msgid "Use the %{strongStart}Test%{strongEnd} option above to create an event." msgstr "" -msgid "Use the Apple App Store Connect integration to easily connect to the Apple App Store with Fastlane in CI/CD pipelines." -msgstr "" - msgid "Use the Google Play integration to connect to Google Play with fastlane in CI/CD pipelines." msgstr "" @@ -52576,6 +52594,9 @@ msgstr "" msgid "Use the search bar on the top of this page" msgstr "" +msgid "Use this integration to connect to the Apple App Store with fastlane in CI/CD pipelines." +msgstr "" + msgid "Use this section to disable your one-time password authenticator and WebAuthn devices. You can also generate new recovery codes." msgstr "" diff --git a/spec/keeps/helpers/postgres_ai_spec.rb b/spec/keeps/helpers/postgres_ai_spec.rb new file mode 100644 index 00000000000..e3003be3460 --- /dev/null +++ b/spec/keeps/helpers/postgres_ai_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' +require './keeps/helpers/postgres_ai' + +RSpec.describe Keeps::Helpers::PostgresAi, feature_category: :tooling do + let(:connection_string) { 'host=localhost port=1234 user=user dbname=dbname' } + let(:password) { 'password' } + let(:pg_client) { instance_double(PG::Connection) } + + before do + stub_env('POSTGRES_AI_CONNECTION_STRING', connection_string) + stub_env('POSTGRES_AI_PASSWORD', password) + + allow(PG).to receive(:connect).with(connection_string, password: password).and_return(pg_client) + end + + describe '#initialize' do + shared_examples 'no credentials supplied' do + it do + expect { described_class.new }.to raise_error(described_class::Error, "No credentials supplied") + end + end + + context 'with no connection string' do + let(:connection_string) { '' } + + include_examples 'no credentials supplied' + end + + context 'with no password' do + let(:password) { '' } + + include_examples 'no credentials supplied' + end + end + + describe '#fetch_background_migration_status' do + let(:job_class_name) { 'ExampleJob' } + let(:query) do + <<~SQL + SELECT id, created_at, updated_at, finished_at, started_at, status, job_class_name + FROM batched_background_migrations + WHERE job_class_name = $1::text + SQL + end + + let(:query_response) { double } + + subject(:result) { described_class.new.fetch_background_migration_status(job_class_name) } + + it 'fetches background migration data from Postgres AI' do + expect(pg_client).to receive(:exec_params).with(query, [job_class_name]).and_return(query_response) + expect(result).to eq(query_response) + end + end +end diff --git a/spec/lib/gitlab/auth_spec.rb b/spec/lib/gitlab/auth_spec.rb index 020089b3880..9974e24ad50 100644 --- a/spec/lib/gitlab/auth_spec.rb +++ b/spec/lib/gitlab/auth_spec.rb @@ -45,26 +45,26 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate expect(subject.all_available_scopes).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode read_observability write_observability create_runner k8s_proxy ai_features] end - it 'contains for non-admin user all non-default scopes without ADMIN access and without observability scopes and ai_features' do + it 'contains for non-admin user all non-default scopes without ADMIN access and without observability scopes' do user = build_stubbed(:user, admin: false) - expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy] + expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy ai_features] end - it 'contains for admin user all non-default scopes with ADMIN access and without observability scopes and ai_features' do + it 'contains for admin user all non-default scopes with ADMIN access and without observability scopes' do user = build_stubbed(:user, admin: true) - expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode create_runner k8s_proxy] + expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode create_runner k8s_proxy ai_features] end - it 'contains for project all resource bot scopes without ai_features' do - expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy] + it 'contains for project all resource bot scopes' do + expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy ai_features] end it 'contains for group all resource bot scopes' do group = build_stubbed(:group).tap { |g| g.namespace_settings = build_stubbed(:namespace_settings, namespace: g) } - expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy] + expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy ai_features] end it 'contains for unsupported type no scopes' do @@ -75,34 +75,6 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate expect(subject.optional_scopes).to match_array %i[read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode openid profile email read_observability write_observability create_runner k8s_proxy ai_features] end - describe 'ai_features scope' do - let(:resource) { nil } - - subject { described_class.available_scopes_for(resource) } - - context 'when resource is user', 'and user has a group with ai features' do - let(:resource) { build_stubbed(:user) } - - it { is_expected.not_to include(:ai_features) } - end - - context 'when resource is project' do - let(:resource) { build_stubbed(:project) } - - it 'does not include ai_features scope' do - is_expected.not_to include(:ai_features) - end - end - - context 'when resource is group' do - let(:resource) { build_stubbed(:group) } - - it 'does not include ai_features scope' do - is_expected.not_to include(:ai_features) - end - end - end - context 'with observability_tracing feature flag' do context 'when disabled' do before do @@ -114,7 +86,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate g.namespace_settings = build_stubbed(:namespace_settings, namespace: g) end - expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy] + expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy ai_features] end it 'contains for project all resource bot scopes without observability scopes' do @@ -123,7 +95,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate end project = build_stubbed(:project, namespace: group) - expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy] + expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy ai_features] end end @@ -140,17 +112,17 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate end it 'contains for group all resource bot scopes including observability scopes' do - expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy] + expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy ai_features] end it 'contains for admin user all non-default scopes with ADMIN access and without observability scopes' do user = build_stubbed(:user, admin: true) - expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode create_runner k8s_proxy] + expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode create_runner k8s_proxy ai_features] end it 'contains for project all resource bot scopes including observability scopes' do - expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy] + expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy ai_features] end it 'contains for other group all resource bot scopes without observability scopes' do @@ -159,7 +131,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate g.namespace_settings = build_stubbed(:namespace_settings, namespace: g) end - expect(subject.available_scopes_for(other_group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy] + expect(subject.available_scopes_for(other_group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy ai_features] end it 'contains for other project all resource bot scopes without observability scopes' do @@ -169,7 +141,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate end other_project = build_stubbed(:project, namespace: other_group) - expect(subject.available_scopes_for(other_project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy] + expect(subject.available_scopes_for(other_project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy ai_features] end end end diff --git a/spec/models/integrations/field_spec.rb b/spec/models/integrations/field_spec.rb index 49eaecd1b2e..22ad71135e7 100644 --- a/spec/models/integrations/field_spec.rb +++ b/spec/models/integrations/field_spec.rb @@ -186,6 +186,22 @@ RSpec.describe ::Integrations::Field, feature_category: :integrations do end end + describe '#api_type' do + it 'returns String' do + expect(field.api_type).to eq(String) + end + + context 'when type is checkbox' do + before do + attrs[:type] = :checkbox + end + + it 'returns Boolean' do + expect(field.api_type).to eq(::API::Integrations::Boolean) + end + end + end + describe '#key?' do it { is_expected.to be_key(:type) } it { is_expected.not_to be_key(:foo) } diff --git a/spec/tooling/quality/test_level_spec.rb b/spec/tooling/quality/test_level_spec.rb index 6ccd2e46f7b..b6c469929ea 100644 --- a/spec/tooling/quality/test_level_spec.rb +++ b/spec/tooling/quality/test_level_spec.rb @@ -46,7 +46,7 @@ RSpec.describe Quality::TestLevel, feature_category: :tooling do context 'when level is unit' do it 'returns a pattern' do expect(subject.pattern(:unit)) - .to eq("spec/{bin,channels,components,config,contracts,db,dependencies,elastic,elastic_integration,experiments,factories,finders,frontend,graphql,haml_lint,helpers,initializers,lib,metrics_server,models,policies,presenters,rack_servers,replicators,routing,rubocop,scripts,serializers,services,sidekiq,sidekiq_cluster,spam,support_specs,tasks,uploaders,validators,views,workers,tooling}{,/**/}*_spec.rb") + .to eq("spec/{bin,channels,components,config,contracts,db,dependencies,elastic,elastic_integration,experiments,factories,finders,frontend,graphql,haml_lint,helpers,initializers,keeps,lib,metrics_server,models,policies,presenters,rack_servers,replicators,routing,rubocop,scripts,serializers,services,sidekiq,sidekiq_cluster,spam,support_specs,tasks,uploaders,validators,views,workers,tooling}{,/**/}*_spec.rb") end end @@ -121,7 +121,7 @@ RSpec.describe Quality::TestLevel, feature_category: :tooling do context 'when level is unit' do it 'returns a regexp' do expect(subject.regexp(:unit)) - .to eq(%r{spec/(bin|channels|components|config|contracts|db|dependencies|elastic|elastic_integration|experiments|factories|finders|frontend|graphql|haml_lint|helpers|initializers|lib|metrics_server|models|policies|presenters|rack_servers|replicators|routing|rubocop|scripts|serializers|services|sidekiq|sidekiq_cluster|spam|support_specs|tasks|uploaders|validators|views|workers|tooling)/}) + .to eq(%r{spec/(bin|channels|components|config|contracts|db|dependencies|elastic|elastic_integration|experiments|factories|finders|frontend|graphql|haml_lint|helpers|initializers|keeps|lib|metrics_server|models|policies|presenters|rack_servers|replicators|routing|rubocop|scripts|serializers|services|sidekiq|sidekiq_cluster|spam|support_specs|tasks|uploaders|validators|views|workers|tooling)/}) end end @@ -230,6 +230,10 @@ RSpec.describe Quality::TestLevel, feature_category: :tooling do expect(subject.level_for('spec/features/abuse_report_spec.rb')).to eq(:system) end + it 'returns the correct level for a keep test' do + expect(subject.level_for('spec/keeps/helpers/postgres_ai_spec.rb')).to eq(:unit) + end + it 'raises an error for an unknown level' do expect { subject.level_for('spec/unknown/foo_spec.rb') } .to raise_error(described_class::UnknownTestLevelError, diff --git a/tooling/quality/test_level.rb b/tooling/quality/test_level.rb index 20e00763f65..c0cc89380d1 100644 --- a/tooling/quality/test_level.rb +++ b/tooling/quality/test_level.rb @@ -33,6 +33,7 @@ module Quality haml_lint helpers initializers + keeps lib metrics_server models