root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2024-08-12 12:07:26 +00:00
parent c1f5a26e33
commit fc3cef14da
30 changed files with 449 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitlab/ci/as-if-foss.gitlab-ci.yml
View File

@ -12,6 +12,7 @@ prepare-as-if-foss-branch:
timeout: 10m
environment:
name: "as-if-foss/${CI_MERGE_REQUEST_IID}"
auto_stop_in: 2 weeks
on_stop: delete-as-if-foss-branch
before_script:
- git clone --single-branch --branch master "${FOSS_REPOSITORY}" gitlab-foss

1
.gitlab/ci/review-apps/main.gitlab-ci.yml
View File

@ -71,6 +71,7 @@ review-build-cng:
environment:
name: review/${CI_COMMIT_REF_SLUG}${SCHEDULE_TYPE} # No separator for SCHEDULE_TYPE so it's compatible as before and looks nice without it
url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}
auto_stop_in: 2 weeks
on_stop: trigger-review-stop
review-deploy:

2
Gemfile.checksum
View File

@ -650,7 +650,7 @@
{"name":"sd_notify","version":"0.1.1","platform":"ruby","checksum":"cbc7ac6caa7cedd26b30a72b5eeb6f36050dc0752df263452ea24fb5a4ad3131"},
{"name":"seed-fu","version":"2.3.7","platform":"ruby","checksum":"f19673443e9af799b730e3d4eca6a89b39e5a36825015dffd00d02ea3365cf74"},
{"name":"selenium-webdriver","version":"4.23.0","platform":"ruby","checksum":"490aeddee879cfea58a4db6628338d60a905bc56cd5e1a60dfbaa9090a19b801"},
{"name":"semver_dialects","version":"3.4.0","platform":"ruby","checksum":"9625fd343cd47335961ccd71249ad905b1a7c03c514a031b35540bc2946eab59"},
{"name":"semver_dialects","version":"3.4.3","platform":"ruby","checksum":"ae3ea99f7806693ab031df3121017c102f1a35f4fc2524674055cb446fb9cc82"},
{"name":"sentry-rails","version":"5.18.2","platform":"ruby","checksum":"4dadf6e7567153e868028d6663a4a88f0cc176b8f30f64b564223df9ee800e27"},
{"name":"sentry-ruby","version":"5.18.2","platform":"ruby","checksum":"df25cb1bd0c6b23f3266e4a77e9b8efd76d41e1680b07b17d257a99262c10d99"},
{"name":"sentry-sidekiq","version":"5.18.2","platform":"ruby","checksum":"86d6a303c7d32c0ced214279111000bd65e11384ed8bd62dcf6580c6a3f75261"},

2
Gemfile.lock
View File

@ -1694,7 +1694,7 @@ GEM
rexml (~> 3.2, >= 3.2.5)
rubyzip (>= 1.2.2, < 3.0)
websocket (~> 1.0)
semver_dialects (3.4.0)
semver_dialects (3.4.3)
deb_version (~> 1.0.1)
pastel (~> 0.8.0)
thor (~> 1.3)

6
Gemfile.next.checksum
View File

@ -662,9 +662,9 @@
{"name":"seed-fu","version":"2.3.7","platform":"ruby","checksum":"f19673443e9af799b730e3d4eca6a89b39e5a36825015dffd00d02ea3365cf74"},
{"name":"selenium-webdriver","version":"4.23.0","platform":"ruby","checksum":"490aeddee879cfea58a4db6628338d60a905bc56cd5e1a60dfbaa9090a19b801"},
{"name":"semver_dialects","version":"3.4.0","platform":"ruby","checksum":"9625fd343cd47335961ccd71249ad905b1a7c03c514a031b35540bc2946eab59"},
{"name":"sentry-rails","version":"5.17.3","platform":"ruby","checksum":"017771c42d739c0ad2213a581ca9d005cf543227bc13662cd1ca9909f2429459"},
{"name":"sentry-ruby","version":"5.17.3","platform":"ruby","checksum":"61791a4b0bb0f95cd87aceeaa1efa6d4ab34d64236c9d5df820478adfe2fbbfc"},
{"name":"sentry-sidekiq","version":"5.17.3","platform":"ruby","checksum":"d0714a218999e41e38127d0c174e0ee62a32b069f92e85b544e0c2125eca2c58"},
{"name":"sentry-rails","version":"5.18.2","platform":"ruby","checksum":"4dadf6e7567153e868028d6663a4a88f0cc176b8f30f64b564223df9ee800e27"},
{"name":"sentry-ruby","version":"5.18.2","platform":"ruby","checksum":"df25cb1bd0c6b23f3266e4a77e9b8efd76d41e1680b07b17d257a99262c10d99"},
{"name":"sentry-sidekiq","version":"5.18.2","platform":"ruby","checksum":"86d6a303c7d32c0ced214279111000bd65e11384ed8bd62dcf6580c6a3f75261"},
{"name":"shellany","version":"0.0.1","platform":"ruby","checksum":"0e127a9132698766d7e752e82cdac8250b6adbd09e6c0a7fbbb6f61964fedee7"},
{"name":"shoulda-matchers","version":"5.1.0","platform":"ruby","checksum":"a01d20589989e9653ab4a28c67d9db2b82bcf0a2496cf01d5e1a95a4aaaf5b07"},
{"name":"sidekiq-cron","version":"1.12.0","platform":"ruby","checksum":"6663080a454088bd88773a0da3ae91e554b8a2e8b06cfc629529a83fd1a3096c"},

16
Gemfile.next.lock
View File

@ -1716,14 +1716,14 @@ GEM
pastel (~> 0.8.0)
thor (~> 1.3)
tty-command (~> 0.10.1)
sentry-rails (5.17.3)
sentry-rails (5.18.2)
railties (>= 5.0)
sentry-ruby (~> 5.17.3)
sentry-ruby (5.17.3)
sentry-ruby (~> 5.18.2)
sentry-ruby (5.18.2)
bigdecimal
concurrent-ruby (~> 1.0, >= 1.0.2)
sentry-sidekiq (5.17.3)
sentry-ruby (~> 5.17.3)
sentry-sidekiq (5.18.2)
sentry-ruby (~> 5.18.2)
sidekiq (>= 3.0)
shellany (0.0.1)
shoulda-matchers (5.1.0)
@ -2280,9 +2280,9 @@ DEPENDENCIES
seed-fu (~> 2.3.7)
selenium-webdriver (~> 4.21, >= 4.21.1)
semver_dialects (~> 3.0)
sentry-rails (~> 5.17.3)
sentry-ruby (~> 5.17.3)
sentry-sidekiq (~> 5.17.3)
sentry-rails (~> 5.18.0)
sentry-ruby (~> 5.18.0)
sentry-sidekiq (~> 5.18.0)
shoulda-matchers (~> 5.1.0)
sidekiq!
sidekiq-cron (~> 1.12.0)

4
app/assets/javascripts/projects/settings/api/access_dropdown_api.js
View File

@ -1,10 +1,10 @@
import axios from '~/lib/utils/axios_utils';
const USERS_PATH = '/-/autocomplete/users.json';
const GROUPS_PATH = '/-/autocomplete/project_groups.json';
export const GROUPS_PATH = '/-/autocomplete/project_groups.json';
const DEPLOY_KEYS_PATH = '/-/autocomplete/deploy_keys_with_owners.json';
const buildUrl = (urlRoot, url) => {
export const buildUrl = (urlRoot, url) => {
let newUrl;
if (urlRoot != null) {
newUrl = urlRoot.replace(/\/$/, '') + url;

3
app/assets/javascripts/projects/settings/branch_rules/components/view/index.vue
View File

@ -66,6 +66,9 @@ export default {
projectPath: {
default: '',
},
projectId: {
default: null,
},
protectedBranchesPath: {
default: '',
},

9
app/assets/javascripts/projects/settings/branch_rules/components/view/rule_drawer.vue
View File

@ -3,6 +3,7 @@ import { GlDrawer, GlButton, GlFormGroup, GlFormCheckbox } from '@gitlab/ui';
import { __ } from '~/locale';
import { DRAWER_Z_INDEX } from '~/lib/utils/constants';
import { getContentWrapperHeight } from '~/lib/utils/dom_utils';
import { USERS_TYPE, GROUPS_TYPE } from '~/vue_shared/components/list_selector/constants';
import { convertToGraphQLId, getIdFromGraphQLId } from '~/graphql_shared/utils';
import {
ACCESS_LEVEL_DEVELOPER_INTEGER,
@ -20,6 +21,8 @@ export default {
ACCESS_LEVEL_MAINTAINER_INTEGER,
ACCESS_LEVEL_ADMIN_INTEGER,
ACCESS_LEVEL_NO_ACCESS_INTEGER,
USERS_TYPE,
GROUPS_TYPE,
i18n: {
saveChanges: __('Save changes'),
cancel: __('Cancel'),
@ -194,16 +197,14 @@ export default {
</gl-form-checkbox>
<items-selector
type="users"
:type="$options.USERS_TYPE"
:items="formatItemsIds(users)"
:users-options="$options.projectUsersOptions"
data-testid="users-selector"
@change="handleRuleDataUpdate('updatedUsers', $event)"
/>
<items-selector
type="groups"
disable-namespace-dropdown
is-project-scoped
:type="$options.GROUPS_TYPE"
:items="formatItemsIds(groups)"
data-testid="groups-selector"
@change="handleRuleDataUpdate('updatedGroups', $event)"

2
app/assets/javascripts/projects/settings/branch_rules/mount_branch_rules.js
View File

@ -17,6 +17,7 @@ export default function mountBranchRules(el, store) {
const {
projectPath,
projectId,
protectedBranchesPath,
branchRulesPath,
approvalRulesPath,
@ -33,6 +34,7 @@ export default function mountBranchRules(el, store) {
apolloProvider,
provide: {
projectPath,
projectId: parseInt(projectId, 10),
branchRulesPath,
protectedBranchesPath,
approvalRulesPath,

20
app/assets/javascripts/vue_shared/components/list_selector/api.js
View File

@ -1,9 +1,11 @@
import Api from '~/api';
import { getProjects } from '~/rest_api';
import axios from '~/lib/utils/axios_utils';
import { convertObjectPropsToCamelCase } from '~/lib/utils/common_utils';
import { ACCESS_LEVEL_REPORTER_INTEGER } from '~/access_level/constants';
import { getIdFromGraphQLId } from '~/graphql_shared/utils';
import groupsAutocompleteQuery from '~/graphql_shared/queries/groups_autocomplete.query.graphql';
import { buildUrl, GROUPS_PATH } from '~/projects/settings/api/access_dropdown_api';
export const fetchProjectGroups = (projectPath, search) => {
return Api.projectGroups(projectPath, {
@ -36,6 +38,24 @@ export const fetchAllGroups = async (apollo, search) => {
);
};
export const fetchGroupsWithProjectAccess = (projectId, search) => {
return axios
.get(buildUrl(gon.relative_url_root || '', GROUPS_PATH), {
params: {
project_id: projectId,
with_project_access: true,
search,
},
})
.then(({ data }) =>
data.map((group) => ({
text: group.name,
value: group.name,
...convertObjectPropsToCamelCase(group),
})),
);
};
export const fetchProjects = async (search) => {
const response = await getProjects(search, { membership: false });
const projects = response?.data || [];

5
app/assets/javascripts/vue_shared/components/list_selector/constants.js
View File

@ -4,6 +4,11 @@ import GroupItem from './group_item.vue';
import DeployKeyItem from './deploy_key_item.vue';
import ProjectItem from './project_item.vue';
export const USERS_TYPE = 'users';
export const GROUPS_TYPE = 'groups';
export const DEPLOY_KEYS_TYPE = 'deployKeys';
export const PROJECTS_TYPE = 'projects';
export const CONFIG = {
users: {
title: __('Users'),

34
app/assets/javascripts/vue_shared/components/list_selector/index.vue
View File

@ -7,6 +7,7 @@ import { __, sprintf } from '~/locale';
import {
fetchProjectGroups,
fetchAllGroups,
fetchGroupsWithProjectAccess,
fetchProjects,
fetchUsers,
} from '~/vue_shared/components/list_selector/api';
@ -18,6 +19,27 @@ const I18N = {
apiErrorMessage: __('An error occurred while fetching. Please try again.'),
};
/**
* Renders a selector and displays a list of selected items.
* Selected items can be:
* - users
* - projects
* - groups
* - deploy keys
*
*
* For groups type, there are three different APIs you can use:
* - `fetchAllGroups()` (default)
* - uses GraphQL `groupsAutocompleteQuery`
* - `fetchProjectGroups()`
* - when `isProjectNamespace` equals `true`,
* - uses `Api.projectGroups()` with parameters `{with_shared: true, shared_min_access_level: ACCESS_LEVEL_REPORTER_INTEGER}`
* - `fetchGroupsWithProjectAccess()`
* - when `isGroupsWithProjectAccess` equals `true`,
* - GET Request `autocomplete/project_groups.json` to fetch groups invited to the project
*
*/
export default {
name: 'ListSelector',
i18n: I18N,
@ -46,6 +68,11 @@ export default {
required: false,
default: null,
},
projectId: {
type: Number,
required: false,
default: null,
},
autofocus: {
type: Boolean,
required: false,
@ -66,6 +93,11 @@ export default {
required: false,
default: false,
},
isGroupsWithProjectAccess: {
type: Boolean,
required: false,
default: false,
},
},
data() {
return {
@ -135,6 +167,8 @@ export default {
let groups = [];
if (parseBoolean(this.isProjectNamespace)) {
groups = await fetchProjectGroups(this.projectPath, search);
} else if (this.isGroupsWithProjectAccess) {
groups = await fetchGroupsWithProjectAccess(this.projectId, search);
} else {
groups = await fetchAllGroups(this.$apollo, search);
}

2
app/models/ci/finished_pipeline_ch_sync_event.rb
View File

@ -8,7 +8,7 @@ module Ci
include PartitionedTable
PARTITION_DURATION = 1.day
PARTITION_CLEANUP_THRESHOLD = 1.year
PARTITION_CLEANUP_THRESHOLD = 30.days
self.table_name = :p_ci_finished_pipeline_ch_sync_events
self.primary_key = :pipeline_id

4
config/gitlab_loose_foreign_keys.yml
View File

@ -208,6 +208,10 @@ dast_site_tokens:
- table: projects
column: project_id
on_delete: async_delete
dast_site_validations:
- table: projects
column: project_id
on_delete: async_delete
deployment_clusters:
- table: clusters
column: cluster_id

2
db/docs/dast_site_validations.yml
View File

@ -7,7 +7,7 @@ feature_categories:
description: The site to be validated with a dast_site_token
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41639
milestone: '13.4'
gitlab_schema: gitlab_main_cell
gitlab_schema: gitlab_sec
allow_cross_foreign_keys:
- gitlab_main_clusterwide
desired_sharding_key:

2
db/post_migrate/20240607105705_queue_backfill_dast_site_validations_project_id.rb
View File

@ -2,7 +2,7 @@
class QueueBackfillDastSiteValidationsProjectId < Gitlab::Database::Migration[2.2]
milestone '17.1'
restrict_gitlab_migration gitlab_schema: :gitlab_main_cell
restrict_gitlab_migration gitlab_schema: :gitlab_sec
MIGRATION = "BackfillDastSiteValidationsProjectId"
DELAY_INTERVAL = 2.minutes

21
db/post_migrate/20240729054128_remove_projects_dast_site_validations_project_id_fk.rb Normal file
View File

@ -0,0 +1,21 @@
# frozen_string_literal: true
class RemoveProjectsDastSiteValidationsProjectIdFk < Gitlab::Database::Migration[2.2]
milestone '17.3'
disable_ddl_transaction!
FOREIGN_KEY_NAME = "fk_20d44ecf03"
def up
with_lock_retries do
remove_foreign_key_if_exists(:dast_site_validations, :projects,
name: FOREIGN_KEY_NAME, reverse_lock_order: true)
end
end
def down
add_concurrent_foreign_key(:dast_site_validations, :projects,
name: FOREIGN_KEY_NAME, column: :project_id,
target_column: :id, on_delete: :cascade)
end
end

1
db/schema_migrations/20240729054128 Normal file
View File

@ -0,0 +1 @@
c1da34c84d6ce71de3002a2ae047e9592c1ae00e81e6af5fb9f947639e6054c1

3
db/structure.sql
View File

@ -32595,9 +32595,6 @@ ALTER TABLE ONLY coverage_fuzzing_corpuses
ALTER TABLE ONLY namespace_settings
ADD CONSTRAINT fk_20cf0eb2f9 FOREIGN KEY (default_compliance_framework_id) REFERENCES compliance_management_frameworks(id) ON DELETE SET NULL;
ALTER TABLE ONLY dast_site_validations
ADD CONSTRAINT fk_20d44ecf03 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
ALTER TABLE ONLY ci_build_trace_metadata
ADD CONSTRAINT fk_21d25cac1a_p FOREIGN KEY (partition_id, trace_artifact_id) REFERENCES p_ci_job_artifacts(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE;

4
doc/administration/geo/replication/troubleshooting/failover.md
View File

@ -1,5 +1,5 @@
---
redirect_to: (../../disaster_recovery/failover_troubleshooting.md)
redirect_to: '../../disaster_recovery/failover_troubleshooting.md'
remove_date: '2025-08-02'
---
@ -8,4 +8,4 @@ This document was moved to [another location](../../disaster_recovery/failover_t
<!-- This redirect file can be deleted after <YYYY-MM-DD>. -->
<!-- Redirects that point to other docs in the same project expire in three months. -->
<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. -->
<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->

2
package.json
View File

@ -70,7 +70,7 @@
"@gitlab/favicon-overlay": "2.0.0",
"@gitlab/fonts": "^1.3.0",
"@gitlab/svgs": "3.112.0",
"@gitlab/ui": "88.1.0",
"@gitlab/ui": "88.6.0",
"@gitlab/web-ide": "^0.0.1-dev-20240731185426",
"@mattiasbuelens/web-streams-adapter": "^0.1.0",
"@rails/actioncable": "7.0.8-4",

160
scripts/review_apps/automated_cleanup.rb
View File

@ -7,16 +7,14 @@
# See https://docs.gitlab.com/ee/development/pipelines/internals.html#using-the-gitlab-ruby-gem-in-the-canonical-project.
require 'gitlab'
require 'optparse'
require 'time'
require_relative File.expand_path('../../tooling/lib/tooling/helm3_client.rb', __dir__)
require_relative File.expand_path('../../tooling/lib/tooling/kubernetes_client.rb', __dir__)
module ReviewApps
class AutomatedCleanup
DEPLOYMENTS_PER_PAGE = 100
ENVIRONMENT_PREFIX = {
review_app: 'review/',
docs_review_app: 'review-docs/'
}.freeze
ENVIRONMENTS_PER_PAGE = 100
IGNORED_HELM_ERRORS = [
'transport is closing',
'error upgrading connection',
@ -57,7 +55,6 @@ module ReviewApps
@gitlab_token = gitlab_token
@api_endpoint = api_endpoint
@dry_run = options[:dry_run]
@environments_not_found_count = 0
end
def gitlab
@ -80,81 +77,23 @@ module ReviewApps
@kubernetes ||= Tooling::KubernetesClient.new
end
def perform_gitlab_environment_cleanup!(days_for_delete:)
def perform_gitlab_environment_cleanup!(env_prefix:, days_for_delete:)
puts "Dry-run mode." if dry_run
puts "Checking for Review Apps not updated in the last #{days_for_delete} days..."
puts "Checking for GitLab #{env_prefix} environments without any deployments in them, or deployed more than #{days_for_delete} days ago..."
checked_environments = []
delete_threshold = threshold_time(days: days_for_delete)
deployments_look_back_threshold = threshold_time(days: days_for_delete * 5)
releases_to_delete = []
gitlab.environments(project_path, per_page: ENVIRONMENTS_PER_PAGE, sort: 'desc', search: env_prefix).auto_paginate do |environment|
next unless environment.name.start_with?(env_prefix)
# TODO: Find a way to reset those, so that we can properly delete them.
next if environment.state == 'stopping' # We cannot delete environments in stopping state
# Delete environments via deployments
gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment|
last_deploy = deployment.created_at
deployed_at = Time.parse(last_deploy)
deployments = gitlab.deployments(project_path, environment: environment.name, order_by: :created_at, sort: :desc)
next if deployments.count > 0 && Time.parse(deployments.first.created_at) > delete_threshold
break if deployed_at < deployments_look_back_threshold
environment = deployment.environment
next unless environment
next unless environment.name.start_with?(ENVIRONMENT_PREFIX[:review_app])
next if checked_environments.include?(environment.slug)
if deployed_at < delete_threshold
deleted_environment = delete_environment(environment, deployment)
if deleted_environment
release = Tooling::Helm3Client::Release.new(name: environment.slug, namespace: environment.slug, revision: 1)
releases_to_delete << release
end
end
checked_environments << environment.slug
stop_environment(environment)
delete_environment(environment)
end
delete_stopped_environments(environment_type: :review_app, checked_environments: checked_environments, last_updated_threshold: delete_threshold) do |environment|
releases_to_delete << Tooling::Helm3Client::Release.new(name: environment.slug, namespace: environment.slug, revision: 1, updated: environment.updated_at)
end
delete_helm_releases(releases_to_delete)
end
def perform_gitlab_docs_environment_cleanup!(days_for_stop:, days_for_delete:)
puts "Dry-run mode." if dry_run
puts "Checking for Docs Review Apps not updated in the last #{days_for_stop} days..."
checked_environments = []
stop_threshold = threshold_time(days: days_for_stop)
delete_threshold = threshold_time(days: days_for_delete)
deployments_look_back_threshold = threshold_time(days: days_for_delete * 5)
# Delete environments via deployments
gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment|
last_deploy = deployment.created_at
deployed_at = Time.parse(last_deploy)
break if deployed_at < deployments_look_back_threshold
environment = deployment.environment
next unless environment
next unless environment.name.start_with?(ENVIRONMENT_PREFIX[:docs_review_app])
next if checked_environments.include?(environment.slug)
if deployed_at < stop_threshold
environment_state = fetch_environment(environment)&.state
stop_environment(environment, deployment) if environment_state && environment_state != 'stopped'
end
delete_environment(environment, deployment) if deployed_at < delete_threshold
checked_environments << environment.slug
end
delete_stopped_environments(environment_type: :docs_review_app, checked_environments: checked_environments, last_updated_threshold: delete_threshold)
end
def perform_helm_releases_cleanup!(days:)
@ -190,55 +129,26 @@ module ReviewApps
attr_reader :api_endpoint, :dry_run, :gitlab_token, :project_path
def fetch_environment(environment)
gitlab.environment(project_path, environment.id)
rescue Errno::ETIMEDOUT => ex
puts "Failed to fetch '#{environment.name}' / '#{environment.slug}' (##{environment.id}):\n#{ex.message}"
nil
end
def stop_environment(environment)
return if environment.state == 'stopped' || environment.state == 'stopping'
def delete_environment(environment, deployment = nil)
release_date = deployment ? deployment.created_at : environment.updated_at
print_release_state(subject: 'Review app', release_name: environment.slug, release_date: release_date, action: 'deleting')
gitlab.delete_environment(project_path, environment.id) unless dry_run
rescue Gitlab::Error::NotFound
puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) was not found: ignoring it"
@environments_not_found_count += 1
if @environments_not_found_count >= ENVIRONMENTS_NOT_FOUND_THRESHOLD
raise "At least #{ENVIRONMENTS_NOT_FOUND_THRESHOLD} environments were missing when we tried to delete them. Please investigate"
end
rescue Gitlab::Error::Forbidden
puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it"
rescue Gitlab::Error::InternalServerError
puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) 500 error: ignoring it"
end
def stop_environment(environment, deployment)
print_release_state(subject: 'Review app', release_name: environment.slug, release_date: deployment.created_at, action: 'stopping')
print_release_state(subject: 'GitLab Environment', release_name: environment.slug, release_date: environment.created_at, action: 'stopping')
gitlab.stop_environment(project_path, environment.id) unless dry_run
rescue Gitlab::Error::Forbidden
puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it"
puts "GitLab environment '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it"
end
def delete_stopped_environments(environment_type:, checked_environments:, last_updated_threshold:)
gitlab.environments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc', states: 'stopped', search: ENVIRONMENT_PREFIX[environment_type]).auto_paginate do |environment|
next if skip_environment?(environment: environment, checked_environments: checked_environments, last_updated_threshold: last_updated_threshold, environment_type: environment_type)
def delete_environment(environment)
return if environment.state == 'stopping'
yield environment if delete_environment(environment) && block_given?
checked_environments << environment.slug
end
end
def skip_environment?(environment:, checked_environments:, last_updated_threshold:, environment_type:)
return true unless environment.name.start_with?(ENVIRONMENT_PREFIX[environment_type])
return true if checked_environments.include?(environment.slug)
return true if Time.parse(environment.updated_at) > last_updated_threshold
false
print_release_state(subject: 'GitLab environment', release_name: environment.slug, release_date: environment.created_at, action: 'deleting')
gitlab.delete_environment(project_path, environment.id) unless dry_run
rescue Gitlab::Error::NotFound
puts "GitLab environment '#{environment.name}' / '#{environment.slug}' (##{environment.id}) was not found: ignoring it"
rescue Gitlab::Error::Forbidden
puts "GitLab environment '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it"
rescue Gitlab::Error::InternalServerError
puts "GitLab environment '#{environment.name}' / '#{environment.slug}' (##{environment.id}) 500 error: ignoring it"
end
def helm_releases
@ -298,10 +208,6 @@ if $PROGRAM_NAME == __FILE__
options = ReviewApps::AutomatedCleanup.parse_args(ARGV)
automated_cleanup = ReviewApps::AutomatedCleanup.new(options: options)
timed('Docs Review Apps cleanup') do
automated_cleanup.perform_gitlab_docs_environment_cleanup!(days_for_stop: 20, days_for_delete: 30)
end
puts
timed('Helm releases cleanup') do
@ -310,8 +216,16 @@ if $PROGRAM_NAME == __FILE__
puts
timed('Review Apps cleanup') do
automated_cleanup.perform_gitlab_environment_cleanup!(days_for_delete: 7)
timed('Review Apps Environments cleanup') do
automated_cleanup.perform_gitlab_environment_cleanup!(env_prefix: 'review/', days_for_delete: 7)
end
timed('Docs Review Apps environments cleanup') do
automated_cleanup.perform_gitlab_environment_cleanup!(env_prefix: 'review-docs/', days_for_delete: 30)
end
timed('as-if-foss Environments cleanup') do
automated_cleanup.perform_gitlab_environment_cleanup!(env_prefix: 'as-if-foss/', days_for_delete: 30)
end
puts

57
spec/frontend/vue_shared/components/list_selector/api_spec.js
View File

@ -1,10 +1,13 @@
import MockAdapter from 'axios-mock-adapter';
import Api from '~/api';
import { getProjects } from '~/rest_api';
import { ACCESS_LEVEL_REPORTER_INTEGER } from '~/access_level/constants';
import { getIdFromGraphQLId } from '~/graphql_shared/utils';
import axios from '~/lib/utils/axios_utils';
import {
fetchProjectGroups,
fetchAllGroups,
fetchGroupsWithProjectAccess,
fetchProjects,
fetchUsers,
} from '~/vue_shared/components/list_selector/api';
@ -12,12 +15,13 @@ import {
jest.mock('~/api');
jest.mock('~/rest_api');
jest.mock('~/graphql_shared/utils');
let axiosMock;
const mockProjectPath = 'group/project';
const mockGroupSearch = 'group';
describe('List Selector Utils', () => {
describe('fetchProjectGroups', () => {
const mockSearch = 'group';
const mockApiResponse = [
{ id: 1, full_name: 'Group 1', name: 'group1' },
{ id: 2, full_name: 'Group 2', name: 'group2' },
@ -28,17 +32,17 @@ describe('List Selector Utils', () => {
});
it('calls Api.projectGroups with correct parameters', async () => {
await fetchProjectGroups(mockProjectPath, mockSearch);
await fetchProjectGroups(mockProjectPath, mockGroupSearch);
expect(Api.projectGroups).toHaveBeenCalledWith(mockProjectPath, {
search: mockSearch,
search: mockGroupSearch,
with_shared: true,
shared_min_access_level: ACCESS_LEVEL_REPORTER_INTEGER,
});
});
it('returns formatted group data', async () => {
const result = await fetchProjectGroups(mockProjectPath, mockSearch);
const result = await fetchProjectGroups(mockProjectPath, mockGroupSearch);
expect(result).toEqual([
{ text: 'Group 1', value: 'group1', id: 1, fullName: 'Group 1', name: 'group1' },
@ -51,7 +55,6 @@ describe('List Selector Utils', () => {
const mockApollo = {
query: jest.fn(),
};
const mockSearch = 'search-term';
const mockGraphQLResponse = {
data: {
groups: {
@ -69,16 +72,16 @@ describe('List Selector Utils', () => {
});
it('calls apollo.query with correct parameters', async () => {
await fetchAllGroups(mockApollo, mockSearch);
await fetchAllGroups(mockApollo, mockGroupSearch);
expect(mockApollo.query).toHaveBeenCalledWith({
query: expect.any(Object),
variables: { search: mockSearch },
variables: { search: mockGroupSearch },
});
});
it('returns formatted group data', async () => {
const result = await fetchAllGroups(mockApollo, mockSearch);
const result = await fetchAllGroups(mockApollo, mockGroupSearch);
expect(result).toEqual([
{
@ -101,6 +104,44 @@ describe('List Selector Utils', () => {
});
});
describe('fetchGroupsWithProjectAccess', () => {
const mockProjectId = 7;
const mockUrl = '/-/autocomplete/project_groups.json';
beforeEach(() => {
const mockAxiosResponse = [
{ id: 1, avatar_url: null, name: 'group1' },
{ id: 2, avatar_url: null, name: 'group2' },
];
axiosMock = new MockAdapter(axios);
axiosMock.onGet(mockUrl).replyOnce(200, mockAxiosResponse);
});
afterEach(() => {
axiosMock.restore();
});
it('calls axios.get with correct parameters', async () => {
await fetchGroupsWithProjectAccess(mockProjectId, mockGroupSearch);
expect(axiosMock.history.get.length).toBe(1);
expect(axiosMock.history.get[0].params).toStrictEqual({
project_id: mockProjectId,
with_project_access: true,
search: mockGroupSearch,
});
});
it('returns formatted group data', async () => {
const result = await fetchGroupsWithProjectAccess(mockProjectId, mockGroupSearch);
expect(result).toEqual([
{ text: 'group1', value: 'group1', id: 1, avatarUrl: null, name: 'group1' },
{ text: 'group2', value: 'group2', id: 2, avatarUrl: null, name: 'group2' },
]);
});
});
describe('fetchProjects', () => {
beforeEach(() => {
jest.clearAllMocks();

121
spec/frontend/vue_shared/components/list_selector/index_spec.js
View File

@ -1,9 +1,11 @@
import Vue from 'vue';
import VueApollo from 'vue-apollo';
import { GlIcon, GlCollapsibleListbox, GlSearchBoxByType } from '@gitlab/ui';
import MockAdapter from 'axios-mock-adapter';
import CrudComponent from '~/vue_shared/components/crud_component.vue';
import Api from '~/api';
import RestApi from '~/rest_api';
import axios from '~/lib/utils/axios_utils';
import { createAlert } from '~/alert';
import { getIdFromGraphQLId } from '~/graphql_shared/utils';
import { mountExtended } from 'helpers/vue_test_utils_helper';
@ -19,6 +21,7 @@ import { ACCESS_LEVEL_REPORTER_INTEGER } from '~/access_level/constants';
import { USERS_RESPONSE_MOCK, GROUPS_RESPONSE_MOCK } from './mock_data';
jest.mock('~/alert');
jest.mock('~/api');
jest.mock('~/rest_api', () => ({
getProjects: jest.fn().mockResolvedValue({
data: [
@ -32,6 +35,7 @@ Vue.use(VueApollo);
describe('List Selector spec', () => {
let wrapper;
let fakeApollo;
let axiosMock;
const USERS_MOCK_PROPS = {
projectPath: 'some/project/path',
@ -185,55 +189,90 @@ describe('List Selector spec', () => {
});
describe('searching', () => {
const searchResponse = GROUPS_RESPONSE_MOCK.data.groups.nodes.map((group) => ({
...group,
id: getIdFromGraphQLId(group.id),
}));
describe('for default all groups', () => {
const searchResponse = GROUPS_RESPONSE_MOCK.data.groups.nodes.map((group) => ({
...group,
id: getIdFromGraphQLId(group.id),
}));
const emitSearchInput = async () => {
findSearchBox().vm.$emit('input', search);
await waitForPromises();
};
const emitSearchInput = async () => {
findSearchBox().vm.$emit('input', search);
await waitForPromises();
};
beforeEach(async () => {
findNamespaceDropdown().vm.$emit('select', 'false');
await emitSearchInput();
});
beforeEach(async () => {
findNamespaceDropdown().vm.$emit('select', 'false');
await emitSearchInput();
});
it('calls query with correct variables when Search box receives an input', () => {
expect(groupsAutocompleteQuerySuccess).toHaveBeenCalledWith({
search,
it('calls query with correct variables when Search box receives an input', () => {
expect(groupsAutocompleteQuerySuccess).toHaveBeenCalledWith({
search,
});
});
it('renders a dropdown for the search results', () => {
expect(findSearchResultsDropdown().props()).toMatchObject({
items: searchResponse,
});
});
it('renders a group component for each search result', () => {
expect(findAllGroupComponents().length).toBe(searchResponse.length);
});
it('emits an event when a search result is selected', () => {
const firstSearchResult = searchResponse[0];
findSearchResultsDropdown().vm.$emit('select', firstSearchResult.name);
expect(wrapper.emitted('select')).toEqual([
[
{
__typename: 'Group',
avatarUrl: null,
fullName: 'Flightjs',
id: 33,
name: 'Flightjs',
text: 'Flightjs',
value: 'Flightjs',
type: 'group',
},
],
]);
});
});
it('renders a dropdown for the search results', () => {
expect(findSearchResultsDropdown().props()).toMatchObject({
items: searchResponse,
describe('for groups with project access', () => {
const mockProjectId = 7;
const mockUrl = '/-/autocomplete/project_groups.json';
const mockAxiosResponse = [
{ id: 1, avatar_url: null, name: 'group1' },
{ id: 2, avatar_url: null, name: 'group2' },
];
axiosMock = new MockAdapter(axios);
const emitSearchInput = async () => {
findSearchBox().vm.$emit('input', search);
await waitForPromises();
};
beforeEach(async () => {
createComponent({
...GROUPS_MOCK_PROPS,
isGroupsWithProjectAccess: true,
projectId: mockProjectId,
});
axiosMock.onGet(mockUrl).replyOnce(200, mockAxiosResponse);
await emitSearchInput();
});
});
it('renders a group component for each search result', () => {
expect(findAllGroupComponents().length).toBe(searchResponse.length);
});
it('emits an event when a search result is selected', () => {
const firstSearchResult = searchResponse[0];
findSearchResultsDropdown().vm.$emit('select', firstSearchResult.name);
expect(wrapper.emitted('select')).toEqual([
[
{
__typename: 'Group',
avatarUrl: null,
fullName: 'Flightjs',
id: 33,
name: 'Flightjs',
text: 'Flightjs',
value: 'Flightjs',
type: 'group',
},
],
]);
it('calls query with correct variables when Search box receives an input', () => {
expect(axiosMock.history.get[0].params).toStrictEqual({
project_id: mockProjectId,
with_project_access: true,
search,
});
});
});
});

3
spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb
View File

@ -39,7 +39,8 @@ RSpec.describe 'cross-database foreign keys' do
'vulnerability_export_parts.vulnerability_export_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/473014
'vulnerability_flags.vulnerability_occurrence_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/473014
'vulnerability_finding_links.vulnerability_occurrence_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/473014
'dast_site_validations.dast_site_token_id' # https://gitlab.com/gitlab-org/gitlab/-/issues/474985
'dast_site_validations.dast_site_token_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/474985
'dast_sites.dast_site_validation_id' # https://gitlab.com/gitlab-org/gitlab/-/issues/476657
]
end

2
spec/migrations/20240607105705_queue_backfill_dast_site_validations_project_id_spec.rb
View File

@ -19,7 +19,7 @@ RSpec.describe QueueBackfillDastSiteValidationsProjectId, feature_category: :dyn
interval: described_class::DELAY_INTERVAL,
batch_size: described_class::BATCH_SIZE,
sub_batch_size: described_class::SUB_BATCH_SIZE,
gitlab_schema: :gitlab_main_cell,
gitlab_schema: :gitlab_sec,
job_arguments: [
:project_id,
:dast_site_tokens,

4
spec/models/ci/finished_pipeline_ch_sync_event_spec.rb
View File

@ -115,7 +115,7 @@ RSpec.describe Ci::FinishedPipelineChSyncEvent, type: :model, feature_category:
context 'when almost all the records are too old' do
before do
travel(1.year - 2.minutes)
travel(30.days - 2.minutes)
end
it { is_expected.to eq(false) }
@ -123,7 +123,7 @@ RSpec.describe Ci::FinishedPipelineChSyncEvent, type: :model, feature_category:
context 'when all the records are too old' do
before do
travel(1.year)
travel(30.days)
end
it { is_expected.to eq(true) }

156
spec/scripts/review_apps/automated_cleanup_spec.rb
View File

@ -23,6 +23,7 @@ RSpec.describe ReviewApps::AutomatedCleanup, feature_category: :tooling do
let(:one_day_ago) { (now - (1 * 24 * 3600)) }
let(:two_days_ago) { (now - (2 * 24 * 3600)) }
let(:three_days_ago) { (now - (3 * 24 * 3600)) }
let(:ten_days_ago) { (now - (10 * 24 * 3600)) }
before do
allow(instance).to receive(:gitlab).and_return(gitlab_client)
@ -145,6 +146,161 @@ RSpec.describe ReviewApps::AutomatedCleanup, feature_category: :tooling do
end
end
describe '#perform_gitlab_environment_cleanup!' do
let(:env_prefix) { 'test-prefix/' }
let(:days_for_delete) { 2 }
let(:deployment_created_at) { three_days_ago.to_s }
let(:env_name) { "#{env_prefix}an-env-name" }
let(:env_state) { 'available' }
# rubocop:disable RSpec/VerifiedDoubles -- Internal API resource
let(:environments) do
[double('GitLab Environment',
id: env_name,
name: env_name,
slug: env_name,
state: env_state,
created_at: one_day_ago.to_s)]
end
let(:deployments) do
[double('GitLab Deployment', created_at: deployment_created_at)]
end
subject do
instance.perform_gitlab_environment_cleanup!(
env_prefix: env_prefix,
days_for_delete: days_for_delete
)
end
before do
allow(gitlab_client).to yield_environments(:environments, environments)
allow(gitlab_client).to receive(:deployments).and_return(deployments)
# Silence outputs to stdout
allow(instance).to receive(:puts)
end
def yield_environments(api_method, environments)
messages = receive_message_chain(api_method, :auto_paginate)
environments.inject(messages) do |stub, environment|
stub.and_yield(environment)
end
end
# rubocop:enable RSpec/VerifiedDoubles -- Internal API resource
context 'when the environment is not for a review-app' do
let(:env_name) { 'not-for-a-review-app' }
it 'does not stop the environment' do
expect(gitlab_client).not_to receive(:stop_environment)
subject
end
it 'does not delete the environment' do
expect(gitlab_client).not_to receive(:delete_environment)
subject
end
end
context 'when the environment is for a review-app' do
context 'when there are no deployments' do
let(:deployments) { [] }
context 'when the environment is already stopped' do
let(:env_state) { 'stopped' }
it 'does not stop the environment' do
expect(gitlab_client).not_to receive(:stop_environment)
allow(gitlab_client).to receive(:delete_environment)
subject
end
it 'deletes the environment' do
expect(gitlab_client).to receive(:delete_environment)
subject
end
end
context 'when the environment is not stopped' do
let(:env_state) { 'available' }
it 'stops the environment' do
expect(gitlab_client).to receive(:stop_environment)
allow(gitlab_client).to receive(:delete_environment)
subject
end
it 'deletes the environment' do
allow(gitlab_client).to receive(:stop_environment)
expect(gitlab_client).to receive(:delete_environment)
subject
end
end
end
context 'when the environment state is stopping' do
let(:env_state) { 'stopping' }
it 'does not stop the environment' do
expect(gitlab_client).not_to receive(:stop_environment)
subject
end
it 'does not delete the environment' do
expect(gitlab_client).not_to receive(:delete_environment)
subject
end
end
context 'when there are deployments' do
context 'when the latest deployment happened later than the days_for_delete argument' do
let(:deployment_created_at) { one_day_ago.to_s }
it 'does not stop the environment' do
expect(gitlab_client).not_to receive(:stop_environment)
subject
end
it 'does not delete the environment' do
expect(gitlab_client).not_to receive(:delete_environment)
subject
end
end
context 'when the latest deployment happened earlier than the days_for_delete argument' do
let(:deployment_created_at) { three_days_ago.to_s }
it 'stops the environment' do
expect(gitlab_client).to receive(:stop_environment)
allow(gitlab_client).to receive(:delete_environment)
subject
end
it 'deletes the environment' do
allow(gitlab_client).to receive(:stop_environment)
expect(gitlab_client).to receive(:delete_environment)
subject
end
end
end
end
end
describe '#perform_helm_releases_cleanup!' do
subject { instance.perform_helm_releases_cleanup!(days: days) }

8
yarn.lock
View File

@ -1350,10 +1350,10 @@
resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.112.0.tgz#9789d6766de222762594c8512bc3b32b619dfe6b"
integrity sha512-lemHTryUoeWcOPd1V8B4Iag6uXU0DvEqfKibPS6PhXAsPdMTCObMb7RPEeVvGp0KI/05fWYJ5EpEBDm5nv0EIw==
"@gitlab/ui@88.1.0":
version "88.1.0"
resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-88.1.0.tgz#67eac5bffbfa5acc552e00cf3d8b9dee69d63378"
integrity sha512-y/9MJfAzJlvBykgzPket6/bYxh00B3LfwabTPdBar6Yh8KrIKlM2hD+fZQt7SeZcS31ILyM2kVgVlV4AK9M6Ag==
"@gitlab/ui@88.6.0":
version "88.6.0"
resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-88.6.0.tgz#3322b44e2095f67fc2fab7a77fec3d3c28692f57"
integrity sha512-h36nmo78aTthjc6jxxRPDQIi9Oaw4RIU1GYkLrlr2Cs/Ah06QvXFH52hp2lQd/5VCZau8jHIXDV97NKhS9E2Xw==
dependencies:
"@floating-ui/dom" "1.4.3"
echarts "^5.3.2"