root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2023-12-18 03:11:52 +00:00
parent e22373c9e6
commit 77c803f528
48 changed files with 1487 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Gemfile
View File

@ -463,6 +463,8 @@ group :development, :test do
# See: https://gitlab.com/gitlab-org/frontend/rfcs/-/issues/106
gem 'vite_rails', '~> 3.0.17', feature_category: :shared
gem 'vite_ruby', '~> 3.5.0', feature_category: :shared
gem 'gitlab-housekeeper', path: 'gems/gitlab-housekeeper' # rubocop:todo Gemfile/MissingFeatureCategory
end
group :development, :test, :danger do

8
Gemfile.lock
View File

@ -29,6 +29,13 @@ PATH
gitlab-backup-cli (0.0.1)
thor (~> 1.3)
PATH
remote: gems/gitlab-housekeeper
specs:
gitlab-housekeeper (0.1.0)
httparty
rubocop
PATH
remote: gems/gitlab-http
specs:
@ -1877,6 +1884,7 @@ DEPENDENCIES
gitlab-dangerfiles (~> 4.6.0)
gitlab-experiment (~> 0.9.1)
gitlab-fog-azure-rm (~> 1.8.0)
gitlab-housekeeper!
gitlab-http!
gitlab-labkit (~> 0.34.0)
gitlab-license (~> 2.3)

2
app/assets/javascripts/super_sidebar/components/help_center.vue
View File

@ -251,7 +251,7 @@ export default {
<template #list-item="{ item }">
<span class="gl-display-flex gl-justify-content-space-between gl-align-items-center">
{{ item.text }}
<gl-icon v-if="item.icon" :name="item.icon" class="gl-text-purple-600" />
<gl-icon v-if="item.icon" :name="item.icon" class="gl-text-gray-500" />
</span>
</template>
</gl-disclosure-dropdown-group>

11
app/models/integration.rb
View File

@ -526,6 +526,17 @@ class Integration < ApplicationRecord
fields.reject { _1[:type] == :password || _1[:name] == 'webhook' || (_1.key?(:if) && _1[:if] != true) }.pluck(:name)
end
def self.api_fields
fields.map do |field|
{
required: field.required?,
name: field.name.to_sym,
type: field.api_type,
desc: field.description
}
end
end
def form_fields
fields.reject { _1[:api_only] == true || (_1.key?(:if) && _1[:if] != true) }
end

26
app/models/integrations/apple_app_store.rb
View File

@ -21,21 +21,31 @@ module Integrations
field :app_store_issuer_id,
section: SECTION_TYPE_CONNECTION,
required: true,
title: -> { s_('AppleAppStore|The Apple App Store Connect Issuer ID.') }
title: -> { s_('AppleAppStore|Apple App Store Connect issuer ID') },
description: -> { s_('AppleAppStore|Apple App Store Connect issuer ID.') }
field :app_store_key_id,
section: SECTION_TYPE_CONNECTION,
required: true,
title: -> { s_('AppleAppStore|The Apple App Store Connect Key ID.') }
title: -> { s_('AppleAppStore|Apple App Store Connect key ID') },
description: -> { s_('AppleAppStore|Apple App Store Connect key ID.') }
field :app_store_private_key_file_name, section: SECTION_TYPE_CONNECTION
field :app_store_private_key, api_only: true
field :app_store_private_key_file_name,
description: -> { s_('Apple App Store Connect private key file name.') },
section: SECTION_TYPE_CONNECTION,
required: true
field :app_store_private_key,
description: -> { s_('Apple App Store Connect private key.') },
required: true,
api_only: true
field :app_store_protected_refs,
type: :checkbox,
section: SECTION_TYPE_CONFIGURATION,
title: -> { s_('AppleAppStore|Protected branches and tags only') },
checkbox_label: -> { s_('AppleAppStore|Only set variables on protected branches and tags') }
description: -> { s_('AppleAppStore|Set variables on protected branches and tags only.') },
checkbox_label: -> { s_('AppleAppStore|Set variables on protected branches and tags only.') }
def self.title
'Apple App Store Connect'
@ -55,10 +65,10 @@ module Integrations
# rubocop:disable Layout/LineLength
texts = [
s_("Use the Apple App Store Connect integration to easily connect to the Apple App Store with Fastlane in CI/CD pipelines."),
s_("After the Apple App Store Connect integration is activated, the following protected variables will be created for CI/CD use."),
s_("Use this integration to connect to the Apple App Store with fastlane in CI/CD pipelines."),
s_("After you enable the integration, the following protected variables are created for CI/CD use:"),
variable_list.join('<br>'),
s_(format("To get started, see the <a href='%{url}' target='_blank'>integration documentation</a> for instructions on how to generate App Store Connect credentials, and how to use this integration.", url: Rails.application.routes.url_helpers.help_page_url('user/project/integrations/apple_app_store'))).html_safe
s_(format("For more information, see the <a href='%{url}' target='_blank'>documentation</a>.", url: Rails.application.routes.url_helpers.help_page_url('user/project/integrations/apple_app_store'))).html_safe
]
# rubocop:enable Layout/LineLength

6
app/models/integrations/field.rb
View File

@ -6,7 +6,7 @@ module Integrations
ATTRIBUTES = %i[
section type placeholder choices value checkbox_label
title help if
title help if description
non_empty_password_help
non_empty_password_title
].concat(BOOLEAN_ATTRIBUTES).freeze
@ -60,6 +60,10 @@ module Integrations
define_method("#{type}?") { self[:type] == type }
end
def api_type
checkbox? ? ::API::Integrations::Boolean : String
end
private
attr_reader :attributes

8
config/feature_flags/experiment/use_cloud_connector_lb.yml Normal file
View File

@ -0,0 +1,8 @@
---
name: use_cloud_connector_lb
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139265
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/435142
milestone: '16.8'
type: experiment
group: group::cloud connector
default_enabled: false

8
config/initializers/1_settings.rb
View File

@ -954,6 +954,14 @@ Gitlab.ee do
Settings.suggested_reviewers['secret_file'] ||= Rails.root.join('.gitlab_suggested_reviewers_secret')
end
#
# Cloud connector
#
Gitlab.ee do
Settings['cloud_connector'] = {}
Settings.cloud_connector['base_url'] ||= ENV['CLOUD_CONNECTOR_BASE_URL'] || 'https://cloud.gitlab.com'
end
#
# Zoekt credentials
#

2
config/locales/doorkeeper.en.yml
View File

@ -134,7 +134,7 @@ en:
write_observability:
Grants write access to GitLab Observability.
ai_features:
Grants permission to perform API actions for GitLab Duo.
Grants permission to perform API actions for GitLab Duo. This scope is designed to work with the GitLab Duo Plugin for JetBrains. For all other extensions, see scope requirements.
openid:
Grants permission to authenticate with GitLab using OpenID Connect. Also gives read-only access to the user's profile and group memberships.
sudo:

17
doc/api/integrations.md
View File

@ -78,11 +78,11 @@ Example response:
]
```
## Apple App Store
## Apple App Store Connect
### Set up Apple App Store
### Set up Apple App Store Connect
Set up the Apple App Store integration for a project.
Set up the Apple App Store Connect integration for a project.
```plaintext
PUT /projects/:id/integrations/apple_app_store
@ -92,10 +92,11 @@ Parameters:
| Parameter | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `app_store_issuer_id` | string | true | The Apple App Store Connect Issuer ID. |
| `app_store_key_id` | string | true | The Apple App Store Connect Key ID. |
| `app_store_private_key` | string | true | The Apple App Store Connect Private Key. |
| `app_store_protected_refs` | boolean | false | Set variables only on protected branches and tags. Defaults to `true` (enabled). |
| `app_store_issuer_id` | string | true | Apple App Store Connect issuer ID. |
| `app_store_key_id` | string | true | Apple App Store Connect key ID. |
| `app_store_private_key_file_name` | string | true | Apple App Store Connect private key file name. |
| `app_store_private_key` | string | true | Apple App Store Connect private key. |
| `app_store_protected_refs` | boolean | false | Set variables on protected branches and tags only. |
### Disable Apple App Store
@ -677,7 +678,7 @@ Parameters:
| --------- | ---- | -------- | ----------- |
| `token` | string | true | GitHub API token with `repo:status` OAuth scope. |
| `repository_url` | string | true | GitHub repository URL. |
| `static_context` | boolean | false | Append instance name instead of branch to [status check name](../user/project/integrations/github.md#static-or-dynamic-status-check-names). |
| `static_context` | boolean | false | Append the hostname of your GitLab instance to the [status check name](../user/project/integrations/github.md#static-or-dynamic-status-check-names). |
### Disable GitHub

2
doc/user/group/settings/group_access_tokens.md
View File

@ -152,7 +152,7 @@ The scope determines the actions you can perform when you authenticate with a gr
| `read_repository` | Grants read access (pull) to all repositories within a group. |
| `write_repository` | Grants read and write access (pull and push) to all repositories within a group. |
| `create_runner` | Grants permission to create runners in a group. |
| `ai_features` | Grants permission to perform API actions for GitLab Duo. |
| `ai_features` | Grants permission to perform API actions for GitLab Duo. This scope is designed to work with the GitLab Duo Plugin for JetBrains. For all other extensions, see scope requirements. |
| `k8s_proxy` | Grants permission to perform Kubernetes API calls using the agent for Kubernetes in a group. |
## Enable or disable group access token creation

2
doc/user/profile/personal_access_tokens.md
View File

@ -123,7 +123,7 @@ A personal access token can perform actions based on the assigned scopes.
| `sudo` | Grants permission to perform API actions as any user in the system, when authenticated as an administrator. |
| `admin_mode` | Grants permission to perform API actions as an administrator, when Admin Mode is enabled. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/107875) in GitLab 15.8.) |
| `create_runner` | Grants permission to create runners. |
| `ai_features` | Grants permission to perform API actions for GitLab Duo. |
| `ai_features` | Grants permission to perform API actions for GitLab Duo. This scope is designed to work with the GitLab Duo Plugin for JetBrains. For all other extensions, see scope requirements. |
| `k8s_proxy` | Grants permission to perform Kubernetes API calls using the agent for Kubernetes. |
WARNING:

18
doc/user/project/integrations/apple_app_store.md
View File

@ -4,7 +4,7 @@ group: Import and Integrate
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments
---
# Apple App Store **(FREE ALL)**
# Apple App Store Connect **(FREE ALL)**
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104888) in GitLab 15.8 [with a flag](../../../administration/feature_flags.md) named `apple_app_store_integration`. Disabled by default.
> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/385335) in GitLab 15.10. Feature flag `apple_app_store_integration` removed.
@ -16,9 +16,9 @@ The feature is still in development, but you can:
- [Report a bug](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=report_bug).
- [Share feedback](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=general_feedback).
With the Apple App Store integration, you can configure your CI/CD pipelines to connect to [App Store Connect](https://appstoreconnect.apple.com) to build and release apps for iOS, iPadOS, macOS, tvOS, and watchOS.
With the Apple App Store Connect integration, you can configure your CI/CD pipelines to connect to [App Store Connect](https://appstoreconnect.apple.com) to build and release apps for iOS, iPadOS, macOS, tvOS, and watchOS.
The Apple App Store integration works out of the box with [fastlane](https://fastlane.tools/). You can also use this integration with other build tools.
The Apple App Store Connect integration works out of the box with [fastlane](https://fastlane.tools/). You can also use this integration with other build tools.
## Prerequisites
@ -26,7 +26,7 @@ An Apple ID enrolled in the [Apple Developer Program](https://developer.apple.co
## Configure GitLab
GitLab supports enabling the Apple App Store integration at the project level. Complete these steps in GitLab:
GitLab supports enabling the Apple App Store Connect integration at the project level. Complete these steps in GitLab:
1. In the Apple App Store Connect portal, generate a new private key for your project by following [these instructions](https://developer.apple.com/documentation/appstoreconnectapi/creating_api_keys_for_app_store_connect_api).
1. On the left sidebar, select **Search or go to** and find your project.
@ -36,15 +36,15 @@ GitLab supports enabling the Apple App Store integration at the project level. C
1. Provide the Apple App Store Connect configuration information:
- **Issuer ID**: The Apple App Store Connect issuer ID.
- **Key ID**: The key ID of the generated private key.
- **Private Key**: The generated private key. You can download this key only once.
- **Protected branches and tags only**: Enable to only set variables on protected branches and tags.
- **Private key**: The generated private key. You can download this key only once.
- **Protected branches and tags only**: Enable to set variables on protected branches and tags only.
1. Select **Save changes**.
After the Apple App Store integration is activated:
After you enable the integration:
- The global variables `$APP_STORE_CONNECT_API_KEY_ISSUER_ID`, `$APP_STORE_CONNECT_API_KEY_KEY_ID`, `$APP_STORE_CONNECT_API_KEY_KEY`, and `$APP_STORE_CONNECT_API_KEY_IS_KEY_CONTENT_BASE64` are created for CI/CD use.
- `$APP_STORE_CONNECT_API_KEY_KEY` contains the Base64 encoded Private Key.
- `$APP_STORE_CONNECT_API_KEY_KEY` contains the Base64-encoded private key.
- `$APP_STORE_CONNECT_API_KEY_IS_KEY_CONTENT_BASE64` is always `true`.
## Security considerations
@ -52,7 +52,7 @@ After the Apple App Store integration is activated:
### CI/CD variable security
Malicious code pushed to your `.gitlab-ci.yml` file could compromise your variables, including
`$APP_STORE_CONNECT_API_KEY_KEY`, and send them to a third-party server. For more details, see
`$APP_STORE_CONNECT_API_KEY_KEY`, and send them to a third-party server. For more information, see
[CI/CD variable security](../../../ci/variables/index.md#cicd-variable-security).
## Enable the integration in fastlane

9
doc/user/project/merge_requests/approvals/settings.md
View File

@ -114,7 +114,8 @@ When this field is changed, it can affect all open merge requests depending on t
## Require user re-authentication to approve
> Requiring re-authentication by using SAML authentication for GitLab.com groups [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5981) in GitLab 16.6 [with a flag](../../../../administration/feature_flags.md) named `ff_require_saml_auth_to_approve`. Disabled by default.
> - Requiring re-authentication by using SAML authentication for GitLab.com groups [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5981) in GitLab 16.6 [with a flag](../../../../administration/feature_flags.md) named `ff_require_saml_auth_to_approve`. Disabled by default.
> - Requiring re-authentication by using SAML authentication for self-managed instances [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/431415) in GitLab 16.7 [with a flag](../../../../administration/feature_flags.md) named `ff_require_saml_auth_to_approve`. Disabled by default.
FLAG:
On self-managed GitLab, by default requiring re-authentication by using SAML authentication is not available. To make it available, an administrator can
@ -123,16 +124,18 @@ On self-managed GitLab, by default requiring re-authentication by using SAML aut
You can force potential approvers to first authenticate with either:
- A password.
- SAML. Available on GitLab.com groups only.
- SAML.
This permission enables an electronic signature for approvals, such as the one defined by
[Code of Federal Regulations (CFR) Part 11](https://www.accessdata.fda.gov/scripts/cdrh/cfdocs/cfcfr/CFRSearch.cfm?CFRPart=11&showFR=1&subpartNode=21:1.0.1.1.8.3)):
[Code of Federal Regulations (CFR) Part 11](https://www.accessdata.fda.gov/scripts/cdrh/cfdocs/cfcfr/CFRSearch.cfm?CFRPart=11&showFR=1&subpartNode=21:1.0.1.1.8.3)). This
setting is only available on top-level groups. For more information, see [Settings cascading](#settings-cascading).
1. Enable password authentication and SAML authentication. For more information on:
- Password authentication, see
[sign-in restrictions documentation](../../../../administration/settings/sign_in_restrictions.md#password-authentication-enabled).
- SAML authentication for GitLab.com groups, see
[SAML SSO for GitLab.com groups documentation](../../../../user/group/saml_sso).
- SAML authentication for self-managed instances, see [SAML SSO for self-managed GitLab instances](../../../../integration/saml.md).
1. On the left sidebar, select **Settings > Merge requests**.
1. In the **Merge request approvals** section, scroll to **Approval settings** and
select **Require user re-authentication (password or SAML) to approve**.

2
doc/user/project/settings/project_access_tokens.md
View File

@ -96,7 +96,7 @@ See the warning in [create a project access token](#create-a-project-access-toke
| `read_repository` | Grants read access (pull) to the repository. |
| `write_repository` | Grants read and write access (pull and push) to the repository. |
| `create_runner` | Grants permission to create runners in the project. |
| `ai_features` | Grants permission to perform API actions for GitLab Duo. |
| `ai_features` | Grants permission to perform API actions for GitLab Duo. This scope is designed to work with the GitLab Duo Plugin for JetBrains. For all other extensions, see scope requirements. |
| `k8s_proxy` | Grants permission to perform Kubernetes API calls using the agent for Kubernetes in the project. |
## Enable or disable project access token creation

12
gems/gitlab-housekeeper/.gitignore vendored Normal file
View File

@ -0,0 +1,12 @@
/.bundle/
/.yardoc
/_yardoc/
/coverage/
/doc/
/pkg/
/spec/reports/
/tmp/
# rspec failure tracking
.rspec_status
/*.gem

4
gems/gitlab-housekeeper/.gitlab-ci.yml Normal file
View File

@ -0,0 +1,4 @@
include:
- local: gems/gem.gitlab-ci.yml
inputs:
gem_name: "gitlab-housekeeper"

3
gems/gitlab-housekeeper/.rspec Normal file
View File

@ -0,0 +1,3 @@
--format documentation
--color
--require spec_helper

14
gems/gitlab-housekeeper/.rubocop.yml Normal file
View File

@ -0,0 +1,14 @@
inherit_from:
- ../config/rubocop.yml
Gitlab/HTTParty:
Enabled: false
Rails/Output:
Enabled: false
Gemfile/MissingFeatureCategory:
Enabled: false
Rails/NegateInclude:
Enabled: false

16
gems/gitlab-housekeeper/Gemfile Normal file
View File

@ -0,0 +1,16 @@
# frozen_string_literal: true
source "https://rubygems.org"
# Specify your gem's dependencies in gitlab-housekeeper.gemspec
gemspec
gem "rake", "~> 13.0"
gem "rspec", "~> 3.0"
gem "pry"
gem 'webmock'
group :development, :test do
gem 'gitlab-rspec', path: '../gitlab-rspec'
end

139
gems/gitlab-housekeeper/Gemfile.lock Normal file
View File

@ -0,0 +1,139 @@
PATH
remote: ../gitlab-rspec
specs:
gitlab-rspec (0.1.0)
activerecord (>= 6.1, < 8)
activesupport (>= 6.1, < 8)
rspec (~> 3.0)
PATH
remote: .
specs:
gitlab-housekeeper (0.1.0)
httparty
rubocop
GEM
remote: https://rubygems.org/
specs:
activemodel (7.0.8)
activesupport (= 7.0.8)
activerecord (7.0.8)
activemodel (= 7.0.8)
activesupport (= 7.0.8)
activesupport (7.0.8)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 1.6, < 2)
minitest (>= 5.1)
tzinfo (~> 2.0)
addressable (2.8.5)
public_suffix (>= 2.0.2, < 6.0)
ast (2.4.2)
coderay (1.1.3)
concurrent-ruby (1.2.2)
crack (0.4.3)
safe_yaml (~> 1.0.0)
diff-lcs (1.5.0)
gitlab-styles (10.1.0)
rubocop (~> 1.50.2)
rubocop-graphql (~> 0.18)
rubocop-performance (~> 1.15)
rubocop-rails (~> 2.17)
rubocop-rspec (~> 2.22)
hashdiff (1.0.1)
httparty (0.21.0)
mini_mime (>= 1.0.0)
multi_xml (>= 0.5.2)
i18n (1.14.1)
concurrent-ruby (~> 1.0)
json (2.7.1)
method_source (1.0.0)
mini_mime (1.1.5)
minitest (5.20.0)
multi_xml (0.6.0)
parallel (1.23.0)
parser (3.2.2.4)
ast (~> 2.4.1)
racc
pry (0.14.2)
coderay (~> 1.1)
method_source (~> 1.0)
public_suffix (5.0.3)
racc (1.7.3)
rack (3.0.8)
rainbow (3.1.1)
rake (13.1.0)
regexp_parser (2.8.3)
rexml (3.2.6)
rspec (3.12.0)
rspec-core (~> 3.12.0)
rspec-expectations (~> 3.12.0)
rspec-mocks (~> 3.12.0)
rspec-core (3.12.2)
rspec-support (~> 3.12.0)
rspec-expectations (3.12.3)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.12.0)
rspec-mocks (3.12.6)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.12.0)
rspec-rails (1.3.2)
rack (>= 1.0.0)
rspec (>= 1.3.0)
rspec-support (3.12.1)
rubocop (1.50.2)
json (~> 2.3)
parallel (~> 1.10)
parser (>= 3.2.0.0)
rainbow (>= 2.2.2, < 4.0)
regexp_parser (>= 1.8, < 3.0)
rexml (>= 3.2.5, < 4.0)
rubocop-ast (>= 1.28.0, < 2.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 2.4.0, < 3.0)
rubocop-ast (1.30.0)
parser (>= 3.2.1.0)
rubocop-capybara (2.19.0)
rubocop (~> 1.41)
rubocop-factory_bot (2.24.0)
rubocop (~> 1.33)
rubocop-graphql (0.19.0)
rubocop (>= 0.87, < 2)
rubocop-performance (1.19.1)
rubocop (>= 1.7.0, < 2.0)
rubocop-ast (>= 0.4.0)
rubocop-rails (2.22.2)
activesupport (>= 4.2.0)
rack (>= 1.1)
rubocop (>= 1.33.0, < 2.0)
rubocop-ast (>= 1.30.0, < 2.0)
rubocop-rspec (2.25.0)
rubocop (~> 1.40)
rubocop-capybara (~> 2.17)
rubocop-factory_bot (~> 2.22)
ruby-progressbar (1.13.0)
safe_yaml (1.0.4)
tzinfo (2.0.6)
concurrent-ruby (~> 1.0)
unicode-display_width (2.5.0)
webmock (3.19.1)
addressable (>= 2.8.0)
crack (>= 0.3.2)
hashdiff (>= 0.4.0, < 2.0.0)
PLATFORMS
arm64-darwin-22
DEPENDENCIES
gitlab-housekeeper!
gitlab-rspec!
gitlab-styles
pry
rake (~> 13.0)
rspec (~> 3.0)
rspec-rails
rubocop-rspec
webmock
BUNDLED WITH
2.4.21

34
gems/gitlab-housekeeper/README.md Normal file
View File

@ -0,0 +1,34 @@
# Gitlab::Housekeeper
Housekeeping following https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134487
## Running
Technically you can skip steps 1-2 below if you don't want to create a fork but
it's recommended as using a bot account with no permissions in
`gitlab-org/gitlab` will ensure we can't cause much damage if the script makes
a mistake. The alternative of using your own API token with it's permissions to
`gitlab-org/gitlab` has slightly more risks.
1. Create a fork of `gitlab-org/gitlab` where your MRs will come from
1. Create a project access token for that project
1. Set `housekeeper` remote to the fork you created
```
git remote add housekeeper git@gitlab.com:DylanGriffith/gitlab.git
```
1. Open a Postgres.ai tunnel on localhost port 6305
1. Set the Postgres AI env vars matching the tunnel details for your tunnel
```
export POSTGRES_AI_CONNECTION_STRING='host=localhost port=6305 user=dylan dbname=gitlabhq_dblab'
export POSTGRES_AI_PASSWORD='the-password'
```
1. Set the GitLab client details. Will be used to create MR from housekeeper remote:
```
export HOUSEKEEPER_FORK_PROJECT_ID=52263761 # Same project as housekeeper remote
export HOUSEKEEPER_TARGET_PROJECT_ID=52263761 # Can be 278964 (gitlab-org/gitlab) when ready to create real MRs
export HOUSEKEEPER_GITLAB_API_TOKEN=the-api-token
```
1. Run it:
```
bundle exec gitlab-housekeeper -d -m3 -r keeps/overdue_finalize_background_migration.rb -k Keeps::OverdueFinalizeBackgroundMigration
```

8
gems/gitlab-housekeeper/Rakefile Normal file
View File

@ -0,0 +1,8 @@
# frozen_string_literal: true
require "bundler/gem_tasks"
require "rspec/core/rake_task"
RSpec::Core::RakeTask.new(:spec)
task default: :spec

32
gems/gitlab-housekeeper/bin/gitlab-housekeeper Executable file
View File

@ -0,0 +1,32 @@
#!/usr/bin/env ruby
require "optparse"
require 'gitlab/housekeeper'
options = {}
OptionParser.new do |opts|
opts.banner = 'Creates merge requests that can be inferred from the current state of the codebase'
opts.on('-m=M', '--max-mrs=M', Integer, 'Limit of MRs to create. Defaults to 1.') do |m|
options[:max_mrs] = m
end
opts.on('-d', '--dry-run', 'Dry-run only. Print the MR titles, descriptions and diffs') do
options[:dry_run] = true
end
opts.on('-r lib/foo.rb lib/bar.rb', '--require lib/foo.rb lib/bar.rb', Array, 'Require keeps specified') do |r|
options[:require] = r
end
opts.on('-k OverdueFinalizeBackgroundMigration,AnotherKeep', '--keeps OverdueFinalizeBackgroundMigration,AnotherKeep', Array, 'Require keeps specified') do |k|
options[:keeps] = k
end
opts.on('-h', '--help', 'Prints this help') do
abort opts.to_s
end
end.parse!
Gitlab::Housekeeper::Runner.new(**options).run

29
gems/gitlab-housekeeper/gitlab-housekeeper.gemspec Normal file
View File

@ -0,0 +1,29 @@
# frozen_string_literal: true
require_relative 'lib/gitlab/housekeeper/version'
Gem::Specification.new do |spec|
spec.name = "gitlab-housekeeper"
spec.version = Gitlab::Housekeeper::VERSION
spec.authors = ["group::tenant-scale"]
spec.email = ["engineering@gitlab.com"]
spec.summary = "Gem summary"
spec.description = "Housekeeping following https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134487"
spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper"
spec.license = "MIT"
spec.required_ruby_version = ">= 3.0"
spec.metadata["rubygems_mfa_required"] = "true"
spec.files = Dir['lib/**/*.rb']
spec.require_paths = ["lib"]
spec.executables = ['gitlab-housekeeper']
spec.add_runtime_dependency 'httparty'
spec.add_runtime_dependency 'rubocop'
spec.add_development_dependency 'gitlab-styles'
spec.add_development_dependency 'rspec-rails'
spec.add_development_dependency "rubocop-rspec"
spec.add_development_dependency 'webmock'
end

11
gems/gitlab-housekeeper/lib/gitlab/housekeeper.rb Normal file
View File

@ -0,0 +1,11 @@
# frozen_string_literal: true
require "gitlab/housekeeper/version"
require "gitlab/housekeeper/runner"
module Gitlab
module Housekeeper
Error = Class.new(StandardError)
# Your code goes here...
end
end

81
gems/gitlab-housekeeper/lib/gitlab/housekeeper/git.rb Normal file
View File

@ -0,0 +1,81 @@
# frozen_string_literal: true
require 'logger'
require 'gitlab/housekeeper/shell'
module Gitlab
module Housekeeper
class Git
def initialize(logger:, branch_from: 'master')
@logger = logger
@branch_from = branch_from
end
def commit_in_branch(change)
branch_name = branch_name(change.identifiers)
create_commit(branch_name, change)
branch_name
end
def with_branch_from_branch
stashed = false
current_branch = Shell.execute('git', 'branch', '--show-current').chomp
result = Shell.execute('git', 'stash')
stashed = !result.include?('No local changes to save')
Shell.execute("git", "checkout", @branch_from)
yield
ensure
Shell.execute("git", "checkout", current_branch)
Shell.execute('git', 'stash', 'pop') if stashed
end
def create_commit(branch_name, change)
current_branch = Shell.execute('git', 'branch', '--show-current').chomp
begin
Shell.execute("git", "branch", '-D', branch_name)
rescue Shell::Error # Might not exist yet
end
Shell.execute("git", "checkout", "-b", branch_name)
Shell.execute("git", "add", *change.changed_files)
commit_message = <<~MSG
#{change.title}
#{change.description}
This commit was generated by
[gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139492).
Changelog: other
MSG
Shell.execute("git", "commit", "-m", commit_message)
ensure
Shell.execute("git", "checkout", current_branch)
end
def branch_name(identifiers)
# Hyphen-case each identifier then join together with hyphens.
branch_name = identifiers
.map { |i| i.gsub(/[[:upper:]]/) { |w| "-#{w.downcase}" } }
.join('-')
.delete_prefix("-")
# Truncate if it's too long and add a digest
if branch_name.length > 240
branch_name = branch_name[0...200] + OpenSSL::Digest::SHA256.hexdigest(branch_name)[0...15]
end
branch_name
end
end
end
end

118
gems/gitlab-housekeeper/lib/gitlab/housekeeper/gitlab_client.rb Normal file
View File

@ -0,0 +1,118 @@
# frozen_string_literal: true
require 'httparty'
require 'json'
module Gitlab
module Housekeeper
class GitlabClient
Error = Class.new(StandardError)
def initialize
@token = ENV.fetch("HOUSEKEEPER_GITLAB_API_TOKEN")
@base_uri = 'https://gitlab.com/api/v4'
end
def create_or_update_merge_request(
source_project_id:,
title:,
description:,
source_branch:,
target_branch:,
target_project_id:
)
existing_iid = get_existing_merge_request(
source_project_id: source_project_id,
source_branch: source_branch,
target_branch: target_branch,
target_project_id: target_project_id
)
if existing_iid
update_existing_merge_request(
existing_iid: existing_iid,
title: title,
description: description,
target_project_id: target_project_id
)
else
create_merge_request(
source_project_id: source_project_id,
title: title,
description: description,
source_branch: source_branch,
target_branch: target_branch,
target_project_id: target_project_id
)
end
end
private
def get_existing_merge_request(source_project_id:, source_branch:, target_branch:, target_project_id:)
response = HTTParty.get("#{@base_uri}/projects/#{target_project_id}/merge_requests",
query: {
state: :opened,
source_branch: source_branch,
target_branch: target_branch,
source_project_id: source_project_id
},
headers: {
'Private-Token' => @token
})
unless (200..299).cover?(response.code)
raise Error,
"Failed with response code: #{response.code} and body:\n#{response.body}"
end
data = JSON.parse(response.body)
return nil if data.empty?
iids = data.pluck('iid')
raise Error, "More than one matching MR exists: iids: #{iids.join(',')}" unless data.size == 1
iids.first
end
def create_merge_request(
source_project_id:, title:, description:, source_branch:, target_branch:,
target_project_id:)
response = HTTParty.post("#{@base_uri}/projects/#{source_project_id}/merge_requests", body: {
title: title,
description: description,
source_branch: source_branch,
target_branch: target_branch,
target_project_id: target_project_id
}.to_json,
headers: {
'Private-Token' => @token,
'Content-Type' => 'application/json'
})
return if (200..299).cover?(response.code)
raise Error,
"Failed with response code: #{response.code} and body:\n#{response.body}"
end
def update_existing_merge_request(existing_iid:, title:, description:, target_project_id:)
response = HTTParty.put("#{@base_uri}/projects/#{target_project_id}/merge_requests/#{existing_iid}", body: {
title: title,
description: description
}.to_json,
headers: {
'Private-Token' => @token,
'Content-Type' => 'application/json'
})
return if (200..299).cover?(response.code)
raise Error,
"Failed with response code: #{response.code} and body:\n#{response.body}"
end
end
end
end

8
gems/gitlab-housekeeper/lib/gitlab/housekeeper/keep.rb Normal file
View File

@ -0,0 +1,8 @@
# frozen_string_literal: true
module Gitlab
module Housekeeper
class Keep # rubocop:disable Lint/EmptyClass
end
end
end

101
gems/gitlab-housekeeper/lib/gitlab/housekeeper/runner.rb Normal file
View File

@ -0,0 +1,101 @@
# frozen_string_literal: true
require 'gitlab/housekeeper/keep'
require "gitlab/housekeeper/gitlab_client"
require "gitlab/housekeeper/git"
require 'digest'
module Gitlab
module Housekeeper
Change = Struct.new(:identifiers, :title, :description, :changed_files)
class Runner
def initialize(max_mrs: 1, dry_run: false, require: [], keeps: nil)
@max_mrs = max_mrs
@dry_run = dry_run
@logger = Logger.new($stdout)
require_keeps(require)
@keeps = if keeps
keeps.map { |k| k.is_a?(String) ? k.constantize : k }
else
all_keeps
end
end
def run
created = 0
git.with_branch_from_branch do
@keeps.each do |keep|
keep.new.each do |change|
branch_name = git.commit_in_branch(change)
if @dry_run
dry_run(change, branch_name)
else
create(change, branch_name)
end
created += 1
break if created >= @max_mrs
end
break if created >= @max_mrs
end
end
puts "Housekeeper created #{created} MRs"
end
def git
@git ||= ::Gitlab::Housekeeper::Git.new(logger: @logger)
end
def require_keeps(files)
files.each do |r|
require(Pathname(r).expand_path.to_s)
end
end
def dry_run(change, branch_name)
puts
puts "# #{change.title}"
puts
puts change.description
puts
puts Shell.execute('git', '--no-pager', 'diff', 'master', branch_name, '--', *change.changed_files)
end
def create(change, branch_name)
dry_run(change, branch_name)
Shell.execute('git', 'push', '-f', 'housekeeper', "#{branch_name}:#{branch_name}")
gitlab_client.create_or_update_merge_request(
source_project_id: housekeeper_fork_project_id,
title: change.title,
description: change.description,
source_branch: branch_name,
target_branch: 'master',
target_project_id: housekeeper_target_project_id
)
end
def housekeeper_fork_project_id
ENV.fetch('HOUSEKEEPER_FORK_PROJECT_ID')
end
def housekeeper_target_project_id
ENV.fetch('HOUSEKEEPER_TARGET_PROJECT_ID')
end
def gitlab_client
@gitlab_client ||= GitlabClient.new
end
def all_keeps
@all_keeps ||= ObjectSpace.each_object(Class).select { |klass| klass < Keep }
end
end
end
end

27
gems/gitlab-housekeeper/lib/gitlab/housekeeper/shell.rb Normal file
View File

@ -0,0 +1,27 @@
# frozen_string_literal: true
require 'open3'
module Gitlab
module Housekeeper
class Shell
Error = Class.new(StandardError)
def self.execute(*cmd)
stdin, stdout, stderr, wait_thr = Open3.popen3(*cmd)
stdin.close
out = stdout.read
stdout.close
err = stderr.read
stderr.close
exit_status = wait_thr.value
raise Error, "Failed with #{exit_status}\n#{out}\n#{err}\n" unless exit_status.success?
out + err
end
end
end
end

7
gems/gitlab-housekeeper/lib/gitlab/housekeeper/version.rb Normal file
View File

@ -0,0 +1,7 @@
# frozen_string_literal: true
module Gitlab
module Housekeeper
VERSION = "0.1.0"
end
end

128
gems/gitlab-housekeeper/spec/gitlab/housekeeper/git_spec.rb Normal file
View File

@ -0,0 +1,128 @@
# frozen_string_literal: true
require 'spec_helper'
require 'fileutils'
require 'tmpdir'
# rubocop:disable RSpec/MultipleMemoizedHelpers
RSpec.describe ::Gitlab::Housekeeper::Git do
let(:logger) { instance_double(Logger, info: nil) }
let(:git) { described_class.new(logger: logger) }
let(:repository_path) { Pathname(Dir.mktmpdir) }
let(:test_branch_name) { 'gitlab-housekeeper--test-branch' }
let(:file_in_master) { 'file_in_master.txt' }
let(:file_in_another_branch) { 'file_in_another_branch.txt' }
def setup_master_branch
File.write(file_in_master, 'File already in master!')
::Gitlab::Housekeeper::Shell.execute('git', 'init')
::Gitlab::Housekeeper::Shell.execute('git', 'checkout', '-b', 'master')
::Gitlab::Housekeeper::Shell.execute('git', 'add', file_in_master)
::Gitlab::Housekeeper::Shell.execute('git', 'commit', '-m', 'Initial commit!')
end
def setup_and_checkout_another_branch
::Gitlab::Housekeeper::Shell.execute('git', 'checkout', '-b', 'another-branch')
File.write(file_in_another_branch, 'File in another unrelated branch should not be in new branch!')
::Gitlab::Housekeeper::Shell.execute('git', 'add', file_in_another_branch)
::Gitlab::Housekeeper::Shell.execute('git', 'commit', '-m', 'Commit in unrelated branch should not be included')
end
before do
@previous_dir = Dir.pwd
Dir.chdir(repository_path)
# Make sure there is a master branch with something to branch from
setup_master_branch
setup_and_checkout_another_branch
end
after do
Dir.chdir(@previous_dir) if @previous_dir # rubocop:disable RSpec/InstanceVariable
FileUtils.rm_rf(repository_path)
end
describe '#with_branch_from_branch and #commit_in_branch' do
let(:file_not_to_commit) { repository_path.join('test_file_not_to_commit.txt') }
let(:test_file1) { 'test_file1.txt' }
let(:test_file2) { 'files/test_file2.txt' }
it 'commits the given change details to the given branch name' do
title = "The commit title"
description = <<~COMMIT
The commit description can be
split over multiple lines!
COMMIT
identifiers = %w[GitlabHousekeeper TestBranch]
Dir.mkdir('files')
File.write(test_file1, "Content in file 1!")
File.write(test_file2, "Other content in file 2!")
File.write(file_not_to_commit, 'Do not commit!')
changed_files = [test_file1, test_file2]
change = ::Gitlab::Housekeeper::Change.new(
identifiers,
title,
description,
changed_files
)
branch_name = nil
git.with_branch_from_branch do
branch_name = git.commit_in_branch(change)
end
expect(branch_name).to eq(test_branch_name)
branches = ::Gitlab::Housekeeper::Shell.execute('git', 'branch')
expect(branches).to include(branch_name)
current_commit_on_another_branch = ::Gitlab::Housekeeper::Shell.execute('git', 'show')
expect(current_commit_on_another_branch).to include('Commit in unrelated branch should not be included')
expected = <<~COMMIT
The commit title
The commit description can be
split over multiple lines!
This commit was generated by
[gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139492).
Changelog: other
diff --git a/files/test_file2.txt b/files/test_file2.txt
new file mode 100644
index 0000000..ff205e0
--- /dev/null
+++ b/files/test_file2.txt
@@ -0,0 +1 @@
+Other content in file 2!
\\ No newline at end of file
diff --git a/test_file1.txt b/test_file1.txt
new file mode 100644
index 0000000..8dd3371
--- /dev/null
+++ b/test_file1.txt
@@ -0,0 +1 @@
+Content in file 1!
\\ No newline at end of file
COMMIT
commit = ::Gitlab::Housekeeper::Shell.execute('git', 'show', branch_name).gsub(/\s/, '')
expected_without_whitespace = expected.gsub(/\s/, '')
expect(commit).to include(expected_without_whitespace)
::Gitlab::Housekeeper::Shell.execute('git', 'checkout', branch_name)
expect(File).to exist(file_in_master)
expect(File).not_to exist(file_in_another_branch)
end
end
end
# rubocop:enable RSpec/MultipleMemoizedHelpers

99
gems/gitlab-housekeeper/spec/gitlab/housekeeper/gitlab_client_spec.rb Normal file
View File

@ -0,0 +1,99 @@
# frozen_string_literal: true
require 'spec_helper'
require 'gitlab/housekeeper/gitlab_client'
RSpec.describe ::Gitlab::Housekeeper::GitlabClient do
let(:client) { described_class.new }
describe '#create_or_update_merge_request' do
let(:params) do
{
source_project_id: 123,
title: 'A new merge request!',
description: 'This merge request is pretty good.',
source_branch: 'the-source-branch',
target_branch: 'the-target-branch',
target_project_id: 456
}
end
let(:existing_mrs) { [] }
before do
stub_env('HOUSEKEEPER_GITLAB_API_TOKEN', 'the-api-token')
# Stub the check to see if the merge request already exists
stub_request(:get, "https://gitlab.com/api/v4/projects/456/merge_requests?state=opened&source_branch=the-source-branch&target_branch=the-target-branch&source_project_id=123")
.with(
headers: {
'Private-Token' => 'the-api-token'
}
)
.to_return(status: 200, body: existing_mrs.to_json)
end
it 'calls the GitLab API passing the token' do
stub = stub_request(:post, "https://gitlab.com/api/v4/projects/123/merge_requests")
.with(
body: {
title: "A new merge request!",
description: "This merge request is pretty good.",
source_branch: "the-source-branch",
target_branch: "the-target-branch",
target_project_id: 456
},
headers: {
'Content-Type' => 'application/json',
'Private-Token' => 'the-api-token'
})
.to_return(status: 200, body: "")
client.create_or_update_merge_request(**params)
expect(stub).to have_been_requested
end
context 'when the merge request for the branch already exists' do
let(:existing_mrs) do
[{ iid: 1234 }]
end
it 'updates the merge request' do
stub = stub_request(:put, "https://gitlab.com/api/v4/projects/456/merge_requests/1234")
.with(
body: {
title: "A new merge request!",
description: "This merge request is pretty good."
}.to_json,
headers: {
'Content-Type' => 'application/json',
'Private-Token' => 'the-api-token'
})
.to_return(status: 200, body: "")
client.create_or_update_merge_request(**params)
expect(stub).to have_been_requested
end
context 'when multiple merge requests exist' do
let(:existing_mrs) do
[{ iid: 1234 }, { iid: 5678 }]
end
it 'raises since we do not expect this to be possible' do
expect { client.create_or_update_merge_request(**params) }.to raise_error(described_class::Error)
end
end
end
it 'raises an error when unsuccessful response' do
stub_request(:post, "https://gitlab.com/api/v4/projects/123/merge_requests")
.to_return(status: 400, body: "Real bad error")
expect do
client.create_or_update_merge_request(**params)
end.to raise_error(described_class::Error, a_string_matching('Real bad error'))
end
end
end

104
gems/gitlab-housekeeper/spec/gitlab/housekeeper/runner_spec.rb Normal file
View File

@ -0,0 +1,104 @@
# frozen_string_literal: true
require 'spec_helper'
require 'gitlab/housekeeper/runner'
RSpec.describe ::Gitlab::Housekeeper::Runner do
let(:fake_keep) { double(:fake_keep) } # rubocop:disable RSpec/VerifiedDoubles
let(:change1) do
::Gitlab::Housekeeper::Change.new(
%w[the identifier for the first change],
"The title of MR1",
"The description of the MR",
['change1.txt', 'change2.txt']
)
end
let(:change2) do
::Gitlab::Housekeeper::Change.new(
%w[the identifier for the second change],
"The title of MR2",
"The description of the MR",
['change1.txt', 'change2.txt']
)
end
let(:change3) do
::Gitlab::Housekeeper::Change.new(
%w[the identifier for the third change],
"The title of MR3",
"The description of the MR",
['change1.txt', 'change2.txt']
)
end
before do
fake_keep_instance = double(:fake_keep_instance) # rubocop:disable RSpec/VerifiedDoubles
allow(fake_keep).to receive(:new).and_return(fake_keep_instance)
allow(fake_keep_instance).to receive(:each)
.and_yield(change1)
.and_yield(change2)
.and_yield(change3)
end
describe '#run' do
before do
stub_env('HOUSEKEEPER_FORK_PROJECT_ID', '123')
stub_env('HOUSEKEEPER_TARGET_PROJECT_ID', '456')
end
it 'loops over the keeps and creates MRs limited by max_mrs' do
# Branches get created
git = instance_double(::Gitlab::Housekeeper::Git)
expect(::Gitlab::Housekeeper::Git).to receive(:new)
.and_return(git)
expect(git).to receive(:with_branch_from_branch)
.and_yield
expect(git).to receive(:commit_in_branch).with(change1)
.and_return('the-identifier-for-the-first-change')
expect(git).to receive(:commit_in_branch).with(change2)
.and_return('the-identifier-for-the-second-change')
# Branches get shown and pushed
expect(::Gitlab::Housekeeper::Shell).to receive(:execute)
.with('git', '--no-pager', 'diff', 'master',
'the-identifier-for-the-first-change', '--', 'change1.txt', 'change2.txt')
expect(::Gitlab::Housekeeper::Shell).to receive(:execute)
.with('git', 'push', '-f', 'housekeeper',
'the-identifier-for-the-first-change:the-identifier-for-the-first-change')
expect(::Gitlab::Housekeeper::Shell).to receive(:execute)
.with('git', '--no-pager', 'diff', 'master',
'the-identifier-for-the-second-change', '--', 'change1.txt', 'change2.txt')
expect(::Gitlab::Housekeeper::Shell).to receive(:execute)
.with('git', 'push', '-f', 'housekeeper',
'the-identifier-for-the-second-change:the-identifier-for-the-second-change')
# Merge requests get created
gitlab_client = instance_double(::Gitlab::Housekeeper::GitlabClient)
expect(::Gitlab::Housekeeper::GitlabClient).to receive(:new)
.and_return(gitlab_client)
expect(gitlab_client).to receive(:create_or_update_merge_request)
.with(
source_project_id: '123',
title: 'The title of MR1',
description: 'The description of the MR',
source_branch: 'the-identifier-for-the-first-change',
target_branch: 'master',
target_project_id: '456'
)
expect(gitlab_client).to receive(:create_or_update_merge_request)
.with(
source_project_id: '123',
title: 'The title of MR2',
description: 'The description of the MR',
source_branch: 'the-identifier-for-the-second-change',
target_branch: 'master',
target_project_id: '456'
)
described_class.new(max_mrs: 2, keeps: [fake_keep]).run
end
end
end

24
gems/gitlab-housekeeper/spec/gitlab/housekeeper/shell_spec.rb Normal file
View File

@ -0,0 +1,24 @@
# frozen_string_literal: true
require 'spec_helper'
require 'gitlab/housekeeper/shell'
RSpec.describe ::Gitlab::Housekeeper::Shell do
describe '.execute' do
it 'delegates to popen3 and returns stdout' do
expect(Open3).to receive(:popen3).with('echo', 'hello world')
.and_call_original
expect(described_class.execute('echo', 'hello world')).to eq("hello world\n")
end
it 'raises when result is not successful' do
expect do
described_class.execute('cat', 'definitelynotafile')
end.to raise_error(
described_class::Error,
a_string_matching("cat: definitelynotafile: No such file or directory\n")
)
end
end
end

7
gems/gitlab-housekeeper/spec/gitlab/housekeeper_spec.rb Normal file
View File

@ -0,0 +1,7 @@
# frozen_string_literal: true
RSpec.describe Gitlab::Housekeeper do
it "has a version number" do
expect(Gitlab::Housekeeper::VERSION).not_to be nil
end
end

20
gems/gitlab-housekeeper/spec/spec_helper.rb Normal file
View File

@ -0,0 +1,20 @@
# frozen_string_literal: true
require 'rspec/mocks'
require "gitlab/housekeeper"
require "gitlab/housekeeper/git"
require 'webmock/rspec'
require 'gitlab/rspec/all'
RSpec.configure do |config|
config.include StubENV
# Enable flags like --only-failures and --next-failure
config.example_status_persistence_file_path = ".rspec_status"
# Disable RSpec exposing methods globally on `Module` and `main`
config.disable_monkey_patching!
config.expect_with :rspec do |c|
c.syntax = :expect
end
end

37
keeps/helpers/postgres_ai.rb Normal file
View File

@ -0,0 +1,37 @@
# frozen_string_literal: true
module Keeps
module Helpers
class PostgresAi
Error = Class.new(StandardError)
def initialize
raise Error, "No credentials supplied" unless connection_string.present? && password.present?
end
def fetch_background_migration_status(job_class_name)
query = <<~SQL
SELECT id, created_at, updated_at, finished_at, started_at, status, job_class_name
FROM batched_background_migrations
WHERE job_class_name = $1::text
SQL
pg_client.exec_params(query, [job_class_name])
end
private
def connection_string
ENV["POSTGRES_AI_CONNECTION_STRING"]
end
def password
ENV["POSTGRES_AI_PASSWORD"]
end
def pg_client
@pg_client ||= PG.connect(connection_string, password: password)
end
end
end
end

205
keeps/overdue_finalize_background_migration.rb Normal file
View File

@ -0,0 +1,205 @@
# frozen_string_literal: true
require ::File.expand_path('../config/environment', __dir__)
require_relative '../lib/generators/post_deployment_migration/post_deployment_migration_generator'
require_relative './helpers/postgres_ai'
module Keeps
class OverdueFinalizeBackgroundMigration < ::Gitlab::Housekeeper::Keep
CUTOFF_MILESTONE = '16.4'
def initialize; end
def each
each_batched_background_migration do |migration_yaml_file, migration|
next unless before_cuttoff_milestone?(migration['milestone'])
job_name = migration['migration_job_name']
next if migration_finalized?(job_name)
migration_record = fetch_migration_status(job_name)
next unless migration_record
# Finalize the migration
title = "Finalize migration #{job_name}"
identifiers = [self.class.name.demodulize, job_name]
last_migration_file = last_migration_for_job(job_name)
# rubocop:disable Gitlab/DocUrl -- Not running inside rails application
description = <<~MARKDOWN
This migration was finished at `#{migration_record.finished_at || migration_record.updated_at}`, you can confirm
the status using our
[batched background migration chatops commands](https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#monitor-the-progress-and-status-of-a-batched-background-migration).
To confirm it is finished you can run:
```
/chatops run batched_background_migrations status #{migration_record.id}
```
The last time this background migration was triggered was in [#{last_migration_file}](https://gitlab.com/gitlab-org/gitlab/-/blob/master/#{last_migration_file})
You can read more about the process for finalizing batched background migrations in
https://docs.gitlab.com/ee/development/database/batched_background_migrations.html .
As part of our process we want to ensure all batched background migrations have had at least one
[required stop](https://docs.gitlab.com/ee/development/database/required_stops.html)
to process the migration. Therefore we can finalize any batched background migration that was added before the
last required stop.
This merge request was created using the
[gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139492)
gem.
MARKDOWN
# rubocop:enable Gitlab/DocUrl
queue_method_node = find_queue_method_node(last_migration_file)
# TODO: Can runner figure out what changed during this block?
migration_name = truncate_migration_name("Finalize#{migration['migration_job_name']}")
PostDeploymentMigration::PostDeploymentMigrationGenerator
.source_root('generator_templates/post_deployment_migration/post_deployment_migration/')
generator = ::PostDeploymentMigration::PostDeploymentMigrationGenerator.new([migration_name], force: true)
migration_file = generator.invoke_all.first
changed_files = [migration_file]
add_ensure_call_to_migration(migration_file, queue_method_node, job_name)
::Gitlab::Housekeeper::Shell.execute('rubocop', '-a', migration_file)
digest = Digest::SHA256.hexdigest(generator.migration_number)
digest_file = Pathname.new('db').join('schema_migrations', generator.migration_number.to_s).to_s
File.open(digest_file, 'w') { |f| f.write(digest) }
add_finalized_by_to_yaml(migration_yaml_file, generator.migration_number)
changed_files << digest_file
changed_files << migration_yaml_file
to_create = ::Gitlab::Housekeeper::Change.new(identifiers, title, description, changed_files)
yield(to_create)
end
end
def truncate_migration_name(migration_name)
# File names not allowed to exceed 100 chars due to Cop/FilenameLength so we truncate to 70 because there will be
# underscores added.
migration_name[0...70]
end
def add_finalized_by_to_yaml(yaml_file, migration_number)
content = YAML.load_file(yaml_file)
content['finalized_by'] = migration_number
File.open(yaml_file, 'w') { |f| f.write(YAML.dump(content)) }
end
def last_migration_for_job(job_name)
result = ::Gitlab::Housekeeper::Shell.execute('git', 'grep', '--name-only', "MIGRATION = .#{job_name}.").chomp
result = result.each_line.select do |file|
File.read(file).include?('queue_batched_background_migration')
end.max
raise "Could not find migration for #{job_name}" unless result.present?
result
end
def add_ensure_call_to_migration(file, queue_method_node, job_name)
source = RuboCop::ProcessedSource.new(File.read(file), 3.1)
ast = source.ast
source_buffer = source.buffer
rewriter = Parser::Source::TreeRewriter.new(source_buffer)
up_method = ast.children[2].each_child_node(:def).find do |child|
child.method_name == :up
end
table_name = queue_method_node.children[3]
column_name = queue_method_node.children[4]
job_arguments = queue_method_node.children[5..].select { |s| s.type != :hash } # All remaining non-keyword args
gitlab_schema = ::Gitlab::Database::GitlabSchema.table_schema(table_name.value.to_s)
added_content = <<~RUBY.strip
disable_ddl_transaction!
restrict_gitlab_migration gitlab_schema: :#{gitlab_schema}
def up
ensure_batched_background_migration_is_finished(
job_class_name: '#{job_name}',
table_name: #{table_name.source},
column_name: #{column_name.source},
job_arguments: [#{job_arguments.map(&:source).join(', ')}],
finalize: true
)
end
RUBY
rewriter.replace(up_method.loc.expression, added_content)
content = strip_comments(rewriter.process)
File.write(file, content)
end
def strip_comments(code)
result = []
code.each_line.with_index do |line, index|
result << line unless index > 0 && line.lstrip.start_with?('#')
end
result.join
end
def fetch_migration_status(job_name)
result = postgres_ai.fetch_background_migration_status(job_name)
return unless result.count == 1
migration_model = ::Gitlab::Database::BackgroundMigration::BatchedMigration.new(result.first)
migration_model if migration_model.finished?
end
def postgres_ai
@postgres_ai ||= Keeps::Helpers::PostgresAi.new
end
def migration_finalized?(job_name)
result = `git grep --name-only "#{job_name}"`.chomp
result.each_line.select do |file|
File.read(file.chomp).include?('ensure_batched_background_migration_is_finished')
end.any?
end
def find_queue_method_node(file)
source = RuboCop::ProcessedSource.new(File.read(file), 3.1)
ast = source.ast
up_method = ast.children[2].children.find do |child|
child.def_type? && child.method_name == :up
end
up_method.each_descendant.find do |child|
child && child.send_type? && child.method_name == :queue_batched_background_migration
end
end
def before_cuttoff_milestone?(milestone)
Gem::Version.new(milestone) < Gem::Version.new(CUTOFF_MILESTONE)
end
def each_batched_background_migration
all_batched_background_migration_files.map do |f|
yield(f, YAML.load_file(f))
end
end
def all_batched_background_migration_files
Dir.glob("db/docs/batched_background_migrations/*.yml")
end
end
end

33
lib/api/helpers/integrations_helpers.rb
View File

@ -126,38 +126,7 @@ module API
def self.integrations
{
'apple-app-store' => [
{
required: true,
name: :app_store_issuer_id,
type: String,
desc: 'The Apple App Store Connect Issuer ID'
},
{
required: true,
name: :app_store_key_id,
type: String,
desc: 'The Apple App Store Connect Key ID'
},
{
required: true,
name: :app_store_private_key,
type: String,
desc: 'The Apple App Store Connect Private Key'
},
{
required: true,
name: :app_store_private_key_file_name,
type: String,
desc: 'The Apple App Store Connect Private Key File Name'
},
{
required: false,
name: :app_store_protected_refs,
type: ::Grape::API::Boolean,
desc: 'Only enable for protected refs'
}
],
'apple-app-store' => ::Integrations::AppleAppStore.api_fields,
'asana' => [
{
required: true,

7
lib/gitlab/auth.rb
View File

@ -436,8 +436,7 @@ module Gitlab
end
def unavailable_scopes_for_resource(resource)
unavailable_observability_scopes_for_resource(resource) +
unavailable_ai_features_scopes_for_resource(resource)
unavailable_observability_scopes_for_resource(resource)
end
def unavailable_observability_scopes_for_resource(resource)
@ -447,10 +446,6 @@ module Gitlab
OBSERVABILITY_SCOPES
end
def unavailable_ai_features_scopes_for_resource(_resource)
AI_FEATURES_SCOPES
end
def non_admin_available_scopes
API_SCOPES + REPOSITORY_SCOPES + registry_scopes + OBSERVABILITY_SCOPES + AI_FEATURES_SCOPES
end

47
locale/gitlab.pot
View File

@ -4430,9 +4430,6 @@ msgstr ""
msgid "After it is removed, the fork relationship can only be restored by using the API. This project will no longer be able to receive or send merge requests to the upstream project or other forks."
msgstr ""
msgid "After the Apple App Store Connect integration is activated, the following protected variables will be created for CI/CD use."
msgstr ""
msgid "After the export is complete, download the data file from a notification email or from this page. You can then import the data file from the %{strong_text_start}Create new group%{strong_text_end} page of another GitLab instance."
msgstr ""
@ -4442,6 +4439,9 @@ msgstr ""
msgid "After you enable the integration, the following protected variable is created for CI/CD use:"
msgstr ""
msgid "After you enable the integration, the following protected variables are created for CI/CD use:"
msgstr ""
msgid "After you've reviewed these contribution guidelines, you'll be all set to"
msgstr ""
@ -5856,6 +5856,27 @@ msgstr ""
msgid "Append the comment with %{tableflip}"
msgstr ""
msgid "Append the hostname of your GitLab instance to the status check name."
msgstr ""
msgid "Apple App Store Connect private key file name."
msgstr ""
msgid "Apple App Store Connect private key."
msgstr ""
msgid "AppleAppStore|Apple App Store Connect issuer ID"
msgstr ""
msgid "AppleAppStore|Apple App Store Connect issuer ID."
msgstr ""
msgid "AppleAppStore|Apple App Store Connect key ID"
msgstr ""
msgid "AppleAppStore|Apple App Store Connect key ID."
msgstr ""
msgid "AppleAppStore|Drag your Private Key file here or %{linkStart}click to upload%{linkEnd}."
msgstr ""
@ -5868,16 +5889,10 @@ msgstr ""
msgid "AppleAppStore|Leave empty to use your current Private Key."
msgstr ""
msgid "AppleAppStore|Only set variables on protected branches and tags"
msgstr ""
msgid "AppleAppStore|Protected branches and tags only"
msgstr ""
msgid "AppleAppStore|The Apple App Store Connect Issuer ID."
msgstr ""
msgid "AppleAppStore|The Apple App Store Connect Key ID."
msgid "AppleAppStore|Set variables on protected branches and tags only."
msgstr ""
msgid "AppleAppStore|The Apple App Store Connect Private Key (.p8)"
@ -22054,9 +22069,15 @@ msgstr ""
msgid "GitHub API rate limit exceeded. Try again after %{reset_time}"
msgstr ""
msgid "GitHub API token with `repo:status` OAuth scope."
msgstr ""
msgid "GitHub import"
msgstr ""
msgid "GitHub repository URL."
msgstr ""
msgid "GitHubImporter|*Merged by: %{author} at %{timestamp}*"
msgstr ""
@ -52558,9 +52579,6 @@ msgstr ""
msgid "Use the %{strongStart}Test%{strongEnd} option above to create an event."
msgstr ""
msgid "Use the Apple App Store Connect integration to easily connect to the Apple App Store with Fastlane in CI/CD pipelines."
msgstr ""
msgid "Use the Google Play integration to connect to Google Play with fastlane in CI/CD pipelines."
msgstr ""
@ -52576,6 +52594,9 @@ msgstr ""
msgid "Use the search bar on the top of this page"
msgstr ""
msgid "Use this integration to connect to the Apple App Store with fastlane in CI/CD pipelines."
msgstr ""
msgid "Use this section to disable your one-time password authenticator and WebAuthn devices. You can also generate new recovery codes."
msgstr ""

57
spec/keeps/helpers/postgres_ai_spec.rb Normal file
View File

@ -0,0 +1,57 @@
# frozen_string_literal: true
require 'spec_helper'
require './keeps/helpers/postgres_ai'
RSpec.describe Keeps::Helpers::PostgresAi, feature_category: :tooling do
let(:connection_string) { 'host=localhost port=1234 user=user dbname=dbname' }
let(:password) { 'password' }
let(:pg_client) { instance_double(PG::Connection) }
before do
stub_env('POSTGRES_AI_CONNECTION_STRING', connection_string)
stub_env('POSTGRES_AI_PASSWORD', password)
allow(PG).to receive(:connect).with(connection_string, password: password).and_return(pg_client)
end
describe '#initialize' do
shared_examples 'no credentials supplied' do
it do
expect { described_class.new }.to raise_error(described_class::Error, "No credentials supplied")
end
end
context 'with no connection string' do
let(:connection_string) { '' }
include_examples 'no credentials supplied'
end
context 'with no password' do
let(:password) { '' }
include_examples 'no credentials supplied'
end
end
describe '#fetch_background_migration_status' do
let(:job_class_name) { 'ExampleJob' }
let(:query) do
<<~SQL
SELECT id, created_at, updated_at, finished_at, started_at, status, job_class_name
FROM batched_background_migrations
WHERE job_class_name = $1::text
SQL
end
let(:query_response) { double }
subject(:result) { described_class.new.fetch_background_migration_status(job_class_name) }
it 'fetches background migration data from Postgres AI' do
expect(pg_client).to receive(:exec_params).with(query, [job_class_name]).and_return(query_response)
expect(result).to eq(query_response)
end
end
end

56
spec/lib/gitlab/auth_spec.rb
View File

@ -45,26 +45,26 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate
expect(subject.all_available_scopes).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode read_observability write_observability create_runner k8s_proxy ai_features]
end
it 'contains for non-admin user all non-default scopes without ADMIN access and without observability scopes and ai_features' do
it 'contains for non-admin user all non-default scopes without ADMIN access and without observability scopes' do
user = build_stubbed(:user, admin: false)
expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy]
expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy ai_features]
end
it 'contains for admin user all non-default scopes with ADMIN access and without observability scopes and ai_features' do
it 'contains for admin user all non-default scopes with ADMIN access and without observability scopes' do
user = build_stubbed(:user, admin: true)
expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode create_runner k8s_proxy]
expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode create_runner k8s_proxy ai_features]
end
it 'contains for project all resource bot scopes without ai_features' do
expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy]
it 'contains for project all resource bot scopes' do
expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy ai_features]
end
it 'contains for group all resource bot scopes' do
group = build_stubbed(:group).tap { |g| g.namespace_settings = build_stubbed(:namespace_settings, namespace: g) }
expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy]
expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy ai_features]
end
it 'contains for unsupported type no scopes' do
@ -75,34 +75,6 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate
expect(subject.optional_scopes).to match_array %i[read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode openid profile email read_observability write_observability create_runner k8s_proxy ai_features]
end
describe 'ai_features scope' do
let(:resource) { nil }
subject { described_class.available_scopes_for(resource) }
context 'when resource is user', 'and user has a group with ai features' do
let(:resource) { build_stubbed(:user) }
it { is_expected.not_to include(:ai_features) }
end
context 'when resource is project' do
let(:resource) { build_stubbed(:project) }
it 'does not include ai_features scope' do
is_expected.not_to include(:ai_features)
end
end
context 'when resource is group' do
let(:resource) { build_stubbed(:group) }
it 'does not include ai_features scope' do
is_expected.not_to include(:ai_features)
end
end
end
context 'with observability_tracing feature flag' do
context 'when disabled' do
before do
@ -114,7 +86,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate
g.namespace_settings = build_stubbed(:namespace_settings, namespace: g)
end
expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy]
expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy ai_features]
end
it 'contains for project all resource bot scopes without observability scopes' do
@ -123,7 +95,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate
end
project = build_stubbed(:project, namespace: group)
expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy]
expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy ai_features]
end
end
@ -140,17 +112,17 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate
end
it 'contains for group all resource bot scopes including observability scopes' do
expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy]
expect(subject.available_scopes_for(group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy ai_features]
end
it 'contains for admin user all non-default scopes with ADMIN access and without observability scopes' do
user = build_stubbed(:user, admin: true)
expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode create_runner k8s_proxy]
expect(subject.available_scopes_for(user)).to match_array %i[api read_user read_api read_repository write_repository read_registry write_registry sudo admin_mode create_runner k8s_proxy ai_features]
end
it 'contains for project all resource bot scopes including observability scopes' do
expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy]
expect(subject.available_scopes_for(project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry read_observability write_observability create_runner k8s_proxy ai_features]
end
it 'contains for other group all resource bot scopes without observability scopes' do
@ -159,7 +131,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate
g.namespace_settings = build_stubbed(:namespace_settings, namespace: g)
end
expect(subject.available_scopes_for(other_group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy]
expect(subject.available_scopes_for(other_group)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy ai_features]
end
it 'contains for other project all resource bot scopes without observability scopes' do
@ -169,7 +141,7 @@ RSpec.describe Gitlab::Auth, :use_clean_rails_memory_store_caching, feature_cate
end
other_project = build_stubbed(:project, namespace: other_group)
expect(subject.available_scopes_for(other_project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy]
expect(subject.available_scopes_for(other_project)).to match_array %i[api read_api read_repository write_repository read_registry write_registry create_runner k8s_proxy ai_features]
end
end
end

16
spec/models/integrations/field_spec.rb
View File

@ -186,6 +186,22 @@ RSpec.describe ::Integrations::Field, feature_category: :integrations do
end
end
describe '#api_type' do
it 'returns String' do
expect(field.api_type).to eq(String)
end
context 'when type is checkbox' do
before do
attrs[:type] = :checkbox
end
it 'returns Boolean' do
expect(field.api_type).to eq(::API::Integrations::Boolean)
end
end
end
describe '#key?' do
it { is_expected.to be_key(:type) }
it { is_expected.not_to be_key(:foo) }

8
spec/tooling/quality/test_level_spec.rb
View File

@ -46,7 +46,7 @@ RSpec.describe Quality::TestLevel, feature_category: :tooling do
context 'when level is unit' do
it 'returns a pattern' do
expect(subject.pattern(:unit))
.to eq("spec/{bin,channels,components,config,contracts,db,dependencies,elastic,elastic_integration,experiments,factories,finders,frontend,graphql,haml_lint,helpers,initializers,lib,metrics_server,models,policies,presenters,rack_servers,replicators,routing,rubocop,scripts,serializers,services,sidekiq,sidekiq_cluster,spam,support_specs,tasks,uploaders,validators,views,workers,tooling}{,/**/}*_spec.rb")
.to eq("spec/{bin,channels,components,config,contracts,db,dependencies,elastic,elastic_integration,experiments,factories,finders,frontend,graphql,haml_lint,helpers,initializers,keeps,lib,metrics_server,models,policies,presenters,rack_servers,replicators,routing,rubocop,scripts,serializers,services,sidekiq,sidekiq_cluster,spam,support_specs,tasks,uploaders,validators,views,workers,tooling}{,/**/}*_spec.rb")
end
end
@ -121,7 +121,7 @@ RSpec.describe Quality::TestLevel, feature_category: :tooling do
context 'when level is unit' do
it 'returns a regexp' do
expect(subject.regexp(:unit))
.to eq(%r{spec/(bin|channels|components|config|contracts|db|dependencies|elastic|elastic_integration|experiments|factories|finders|frontend|graphql|haml_lint|helpers|initializers|lib|metrics_server|models|policies|presenters|rack_servers|replicators|routing|rubocop|scripts|serializers|services|sidekiq|sidekiq_cluster|spam|support_specs|tasks|uploaders|validators|views|workers|tooling)/})
.to eq(%r{spec/(bin|channels|components|config|contracts|db|dependencies|elastic|elastic_integration|experiments|factories|finders|frontend|graphql|haml_lint|helpers|initializers|keeps|lib|metrics_server|models|policies|presenters|rack_servers|replicators|routing|rubocop|scripts|serializers|services|sidekiq|sidekiq_cluster|spam|support_specs|tasks|uploaders|validators|views|workers|tooling)/})
end
end
@ -230,6 +230,10 @@ RSpec.describe Quality::TestLevel, feature_category: :tooling do
expect(subject.level_for('spec/features/abuse_report_spec.rb')).to eq(:system)
end
it 'returns the correct level for a keep test' do
expect(subject.level_for('spec/keeps/helpers/postgres_ai_spec.rb')).to eq(:unit)
end
it 'raises an error for an unknown level' do
expect { subject.level_for('spec/unknown/foo_spec.rb') }
.to raise_error(described_class::UnknownTestLevelError,

1
tooling/quality/test_level.rb
View File

@ -33,6 +33,7 @@ module Quality
haml_lint
helpers
initializers
keeps
lib
metrics_server
models