diff --git a/.gitlab/ci/as-if-foss.gitlab-ci.yml b/.gitlab/ci/as-if-foss.gitlab-ci.yml index 63b89c6e992..b0750e36ee0 100644 --- a/.gitlab/ci/as-if-foss.gitlab-ci.yml +++ b/.gitlab/ci/as-if-foss.gitlab-ci.yml @@ -12,6 +12,7 @@ prepare-as-if-foss-branch: timeout: 10m environment: name: "as-if-foss/${CI_MERGE_REQUEST_IID}" + auto_stop_in: 2 weeks on_stop: delete-as-if-foss-branch before_script: - git clone --single-branch --branch master "${FOSS_REPOSITORY}" gitlab-foss diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml index ccfac6365f2..846b4ff7832 100644 --- a/.gitlab/ci/review-apps/main.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml @@ -71,6 +71,7 @@ review-build-cng: environment: name: review/${CI_COMMIT_REF_SLUG}${SCHEDULE_TYPE} # No separator for SCHEDULE_TYPE so it's compatible as before and looks nice without it url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} + auto_stop_in: 2 weeks on_stop: trigger-review-stop review-deploy: diff --git a/Gemfile.checksum b/Gemfile.checksum index d68553ba268..03f585337f3 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -650,7 +650,7 @@ {"name":"sd_notify","version":"0.1.1","platform":"ruby","checksum":"cbc7ac6caa7cedd26b30a72b5eeb6f36050dc0752df263452ea24fb5a4ad3131"}, {"name":"seed-fu","version":"2.3.7","platform":"ruby","checksum":"f19673443e9af799b730e3d4eca6a89b39e5a36825015dffd00d02ea3365cf74"}, {"name":"selenium-webdriver","version":"4.23.0","platform":"ruby","checksum":"490aeddee879cfea58a4db6628338d60a905bc56cd5e1a60dfbaa9090a19b801"}, -{"name":"semver_dialects","version":"3.4.0","platform":"ruby","checksum":"9625fd343cd47335961ccd71249ad905b1a7c03c514a031b35540bc2946eab59"}, +{"name":"semver_dialects","version":"3.4.3","platform":"ruby","checksum":"ae3ea99f7806693ab031df3121017c102f1a35f4fc2524674055cb446fb9cc82"}, {"name":"sentry-rails","version":"5.18.2","platform":"ruby","checksum":"4dadf6e7567153e868028d6663a4a88f0cc176b8f30f64b564223df9ee800e27"}, {"name":"sentry-ruby","version":"5.18.2","platform":"ruby","checksum":"df25cb1bd0c6b23f3266e4a77e9b8efd76d41e1680b07b17d257a99262c10d99"}, {"name":"sentry-sidekiq","version":"5.18.2","platform":"ruby","checksum":"86d6a303c7d32c0ced214279111000bd65e11384ed8bd62dcf6580c6a3f75261"}, diff --git a/Gemfile.lock b/Gemfile.lock index d2e1ce79466..018c2fdf71d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1694,7 +1694,7 @@ GEM rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) - semver_dialects (3.4.0) + semver_dialects (3.4.3) deb_version (~> 1.0.1) pastel (~> 0.8.0) thor (~> 1.3) diff --git a/Gemfile.next.checksum b/Gemfile.next.checksum index b26137d4741..e70ac1423c8 100644 --- a/Gemfile.next.checksum +++ b/Gemfile.next.checksum @@ -662,9 +662,9 @@ {"name":"seed-fu","version":"2.3.7","platform":"ruby","checksum":"f19673443e9af799b730e3d4eca6a89b39e5a36825015dffd00d02ea3365cf74"}, {"name":"selenium-webdriver","version":"4.23.0","platform":"ruby","checksum":"490aeddee879cfea58a4db6628338d60a905bc56cd5e1a60dfbaa9090a19b801"}, {"name":"semver_dialects","version":"3.4.0","platform":"ruby","checksum":"9625fd343cd47335961ccd71249ad905b1a7c03c514a031b35540bc2946eab59"}, -{"name":"sentry-rails","version":"5.17.3","platform":"ruby","checksum":"017771c42d739c0ad2213a581ca9d005cf543227bc13662cd1ca9909f2429459"}, -{"name":"sentry-ruby","version":"5.17.3","platform":"ruby","checksum":"61791a4b0bb0f95cd87aceeaa1efa6d4ab34d64236c9d5df820478adfe2fbbfc"}, -{"name":"sentry-sidekiq","version":"5.17.3","platform":"ruby","checksum":"d0714a218999e41e38127d0c174e0ee62a32b069f92e85b544e0c2125eca2c58"}, +{"name":"sentry-rails","version":"5.18.2","platform":"ruby","checksum":"4dadf6e7567153e868028d6663a4a88f0cc176b8f30f64b564223df9ee800e27"}, +{"name":"sentry-ruby","version":"5.18.2","platform":"ruby","checksum":"df25cb1bd0c6b23f3266e4a77e9b8efd76d41e1680b07b17d257a99262c10d99"}, +{"name":"sentry-sidekiq","version":"5.18.2","platform":"ruby","checksum":"86d6a303c7d32c0ced214279111000bd65e11384ed8bd62dcf6580c6a3f75261"}, {"name":"shellany","version":"0.0.1","platform":"ruby","checksum":"0e127a9132698766d7e752e82cdac8250b6adbd09e6c0a7fbbb6f61964fedee7"}, {"name":"shoulda-matchers","version":"5.1.0","platform":"ruby","checksum":"a01d20589989e9653ab4a28c67d9db2b82bcf0a2496cf01d5e1a95a4aaaf5b07"}, {"name":"sidekiq-cron","version":"1.12.0","platform":"ruby","checksum":"6663080a454088bd88773a0da3ae91e554b8a2e8b06cfc629529a83fd1a3096c"}, diff --git a/Gemfile.next.lock b/Gemfile.next.lock index db3308e1bbe..b952c7fab46 100644 --- a/Gemfile.next.lock +++ b/Gemfile.next.lock @@ -1716,14 +1716,14 @@ GEM pastel (~> 0.8.0) thor (~> 1.3) tty-command (~> 0.10.1) - sentry-rails (5.17.3) + sentry-rails (5.18.2) railties (>= 5.0) - sentry-ruby (~> 5.17.3) - sentry-ruby (5.17.3) + sentry-ruby (~> 5.18.2) + sentry-ruby (5.18.2) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) - sentry-sidekiq (5.17.3) - sentry-ruby (~> 5.17.3) + sentry-sidekiq (5.18.2) + sentry-ruby (~> 5.18.2) sidekiq (>= 3.0) shellany (0.0.1) shoulda-matchers (5.1.0) @@ -2280,9 +2280,9 @@ DEPENDENCIES seed-fu (~> 2.3.7) selenium-webdriver (~> 4.21, >= 4.21.1) semver_dialects (~> 3.0) - sentry-rails (~> 5.17.3) - sentry-ruby (~> 5.17.3) - sentry-sidekiq (~> 5.17.3) + sentry-rails (~> 5.18.0) + sentry-ruby (~> 5.18.0) + sentry-sidekiq (~> 5.18.0) shoulda-matchers (~> 5.1.0) sidekiq! sidekiq-cron (~> 1.12.0) diff --git a/app/assets/javascripts/projects/settings/api/access_dropdown_api.js b/app/assets/javascripts/projects/settings/api/access_dropdown_api.js index 89ae2a82c6d..3ab805b9f15 100644 --- a/app/assets/javascripts/projects/settings/api/access_dropdown_api.js +++ b/app/assets/javascripts/projects/settings/api/access_dropdown_api.js @@ -1,10 +1,10 @@ import axios from '~/lib/utils/axios_utils'; const USERS_PATH = '/-/autocomplete/users.json'; -const GROUPS_PATH = '/-/autocomplete/project_groups.json'; +export const GROUPS_PATH = '/-/autocomplete/project_groups.json'; const DEPLOY_KEYS_PATH = '/-/autocomplete/deploy_keys_with_owners.json'; -const buildUrl = (urlRoot, url) => { +export const buildUrl = (urlRoot, url) => { let newUrl; if (urlRoot != null) { newUrl = urlRoot.replace(/\/$/, '') + url; diff --git a/app/assets/javascripts/projects/settings/branch_rules/components/view/index.vue b/app/assets/javascripts/projects/settings/branch_rules/components/view/index.vue index 51aad79c1aa..ad2d3bb4c3d 100644 --- a/app/assets/javascripts/projects/settings/branch_rules/components/view/index.vue +++ b/app/assets/javascripts/projects/settings/branch_rules/components/view/index.vue @@ -66,6 +66,9 @@ export default { projectPath: { default: '', }, + projectId: { + default: null, + }, protectedBranchesPath: { default: '', }, diff --git a/app/assets/javascripts/projects/settings/branch_rules/components/view/rule_drawer.vue b/app/assets/javascripts/projects/settings/branch_rules/components/view/rule_drawer.vue index a5f596de746..49df8390fbc 100644 --- a/app/assets/javascripts/projects/settings/branch_rules/components/view/rule_drawer.vue +++ b/app/assets/javascripts/projects/settings/branch_rules/components/view/rule_drawer.vue @@ -3,6 +3,7 @@ import { GlDrawer, GlButton, GlFormGroup, GlFormCheckbox } from '@gitlab/ui'; import { __ } from '~/locale'; import { DRAWER_Z_INDEX } from '~/lib/utils/constants'; import { getContentWrapperHeight } from '~/lib/utils/dom_utils'; +import { USERS_TYPE, GROUPS_TYPE } from '~/vue_shared/components/list_selector/constants'; import { convertToGraphQLId, getIdFromGraphQLId } from '~/graphql_shared/utils'; import { ACCESS_LEVEL_DEVELOPER_INTEGER, @@ -20,6 +21,8 @@ export default { ACCESS_LEVEL_MAINTAINER_INTEGER, ACCESS_LEVEL_ADMIN_INTEGER, ACCESS_LEVEL_NO_ACCESS_INTEGER, + USERS_TYPE, + GROUPS_TYPE, i18n: { saveChanges: __('Save changes'), cancel: __('Cancel'), @@ -194,16 +197,14 @@ export default { { return Api.projectGroups(projectPath, { @@ -36,6 +38,24 @@ export const fetchAllGroups = async (apollo, search) => { ); }; +export const fetchGroupsWithProjectAccess = (projectId, search) => { + return axios + .get(buildUrl(gon.relative_url_root || '', GROUPS_PATH), { + params: { + project_id: projectId, + with_project_access: true, + search, + }, + }) + .then(({ data }) => + data.map((group) => ({ + text: group.name, + value: group.name, + ...convertObjectPropsToCamelCase(group), + })), + ); +}; + export const fetchProjects = async (search) => { const response = await getProjects(search, { membership: false }); const projects = response?.data || []; diff --git a/app/assets/javascripts/vue_shared/components/list_selector/constants.js b/app/assets/javascripts/vue_shared/components/list_selector/constants.js index 09cc9a2737b..58807b5a287 100644 --- a/app/assets/javascripts/vue_shared/components/list_selector/constants.js +++ b/app/assets/javascripts/vue_shared/components/list_selector/constants.js @@ -4,6 +4,11 @@ import GroupItem from './group_item.vue'; import DeployKeyItem from './deploy_key_item.vue'; import ProjectItem from './project_item.vue'; +export const USERS_TYPE = 'users'; +export const GROUPS_TYPE = 'groups'; +export const DEPLOY_KEYS_TYPE = 'deployKeys'; +export const PROJECTS_TYPE = 'projects'; + export const CONFIG = { users: { title: __('Users'), diff --git a/app/assets/javascripts/vue_shared/components/list_selector/index.vue b/app/assets/javascripts/vue_shared/components/list_selector/index.vue index 1e2411ad58e..84f69228331 100644 --- a/app/assets/javascripts/vue_shared/components/list_selector/index.vue +++ b/app/assets/javascripts/vue_shared/components/list_selector/index.vue @@ -7,6 +7,7 @@ import { __, sprintf } from '~/locale'; import { fetchProjectGroups, fetchAllGroups, + fetchGroupsWithProjectAccess, fetchProjects, fetchUsers, } from '~/vue_shared/components/list_selector/api'; @@ -18,6 +19,27 @@ const I18N = { apiErrorMessage: __('An error occurred while fetching. Please try again.'), }; +/** + * Renders a selector and displays a list of selected items. + * Selected items can be: + * - users + * - projects + * - groups + * - deploy keys + * + * + * For groups type, there are three different APIs you can use: + * - `fetchAllGroups()` (default) + * - uses GraphQL `groupsAutocompleteQuery` + * - `fetchProjectGroups()` + * - when `isProjectNamespace` equals `true`, + * - uses `Api.projectGroups()` with parameters `{with_shared: true, shared_min_access_level: ACCESS_LEVEL_REPORTER_INTEGER}` + * - `fetchGroupsWithProjectAccess()` + * - when `isGroupsWithProjectAccess` equals `true`, + * - GET Request `autocomplete/project_groups.json` to fetch groups invited to the project + * + */ + export default { name: 'ListSelector', i18n: I18N, @@ -46,6 +68,11 @@ export default { required: false, default: null, }, + projectId: { + type: Number, + required: false, + default: null, + }, autofocus: { type: Boolean, required: false, @@ -66,6 +93,11 @@ export default { required: false, default: false, }, + isGroupsWithProjectAccess: { + type: Boolean, + required: false, + default: false, + }, }, data() { return { @@ -135,6 +167,8 @@ export default { let groups = []; if (parseBoolean(this.isProjectNamespace)) { groups = await fetchProjectGroups(this.projectPath, search); + } else if (this.isGroupsWithProjectAccess) { + groups = await fetchGroupsWithProjectAccess(this.projectId, search); } else { groups = await fetchAllGroups(this.$apollo, search); } diff --git a/app/models/ci/finished_pipeline_ch_sync_event.rb b/app/models/ci/finished_pipeline_ch_sync_event.rb index fce2d1e947f..f1e4331f412 100644 --- a/app/models/ci/finished_pipeline_ch_sync_event.rb +++ b/app/models/ci/finished_pipeline_ch_sync_event.rb @@ -8,7 +8,7 @@ module Ci include PartitionedTable PARTITION_DURATION = 1.day - PARTITION_CLEANUP_THRESHOLD = 1.year + PARTITION_CLEANUP_THRESHOLD = 30.days self.table_name = :p_ci_finished_pipeline_ch_sync_events self.primary_key = :pipeline_id diff --git a/config/gitlab_loose_foreign_keys.yml b/config/gitlab_loose_foreign_keys.yml index 6398e1dbd6d..e534d1edd16 100644 --- a/config/gitlab_loose_foreign_keys.yml +++ b/config/gitlab_loose_foreign_keys.yml @@ -208,6 +208,10 @@ dast_site_tokens: - table: projects column: project_id on_delete: async_delete +dast_site_validations: + - table: projects + column: project_id + on_delete: async_delete deployment_clusters: - table: clusters column: cluster_id diff --git a/db/docs/dast_site_validations.yml b/db/docs/dast_site_validations.yml index da71afd4cbd..a2deacaa0dc 100644 --- a/db/docs/dast_site_validations.yml +++ b/db/docs/dast_site_validations.yml @@ -7,7 +7,7 @@ feature_categories: description: The site to be validated with a dast_site_token introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41639 milestone: '13.4' -gitlab_schema: gitlab_main_cell +gitlab_schema: gitlab_sec allow_cross_foreign_keys: - gitlab_main_clusterwide desired_sharding_key: diff --git a/db/post_migrate/20240607105705_queue_backfill_dast_site_validations_project_id.rb b/db/post_migrate/20240607105705_queue_backfill_dast_site_validations_project_id.rb index d99146e9597..5660f75693a 100644 --- a/db/post_migrate/20240607105705_queue_backfill_dast_site_validations_project_id.rb +++ b/db/post_migrate/20240607105705_queue_backfill_dast_site_validations_project_id.rb @@ -2,7 +2,7 @@ class QueueBackfillDastSiteValidationsProjectId < Gitlab::Database::Migration[2.2] milestone '17.1' - restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + restrict_gitlab_migration gitlab_schema: :gitlab_sec MIGRATION = "BackfillDastSiteValidationsProjectId" DELAY_INTERVAL = 2.minutes diff --git a/db/post_migrate/20240729054128_remove_projects_dast_site_validations_project_id_fk.rb b/db/post_migrate/20240729054128_remove_projects_dast_site_validations_project_id_fk.rb new file mode 100644 index 00000000000..1ec5c8ae8eb --- /dev/null +++ b/db/post_migrate/20240729054128_remove_projects_dast_site_validations_project_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveProjectsDastSiteValidationsProjectIdFk < Gitlab::Database::Migration[2.2] + milestone '17.3' + disable_ddl_transaction! + + FOREIGN_KEY_NAME = "fk_20d44ecf03" + + def up + with_lock_retries do + remove_foreign_key_if_exists(:dast_site_validations, :projects, + name: FOREIGN_KEY_NAME, reverse_lock_order: true) + end + end + + def down + add_concurrent_foreign_key(:dast_site_validations, :projects, + name: FOREIGN_KEY_NAME, column: :project_id, + target_column: :id, on_delete: :cascade) + end +end diff --git a/db/schema_migrations/20240729054128 b/db/schema_migrations/20240729054128 new file mode 100644 index 00000000000..f3e9fc89a61 --- /dev/null +++ b/db/schema_migrations/20240729054128 @@ -0,0 +1 @@ +c1da34c84d6ce71de3002a2ae047e9592c1ae00e81e6af5fb9f947639e6054c1 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 68ef1f143db..3c5aa3a35b8 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -32595,9 +32595,6 @@ ALTER TABLE ONLY coverage_fuzzing_corpuses ALTER TABLE ONLY namespace_settings ADD CONSTRAINT fk_20cf0eb2f9 FOREIGN KEY (default_compliance_framework_id) REFERENCES compliance_management_frameworks(id) ON DELETE SET NULL; -ALTER TABLE ONLY dast_site_validations - ADD CONSTRAINT fk_20d44ecf03 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; - ALTER TABLE ONLY ci_build_trace_metadata ADD CONSTRAINT fk_21d25cac1a_p FOREIGN KEY (partition_id, trace_artifact_id) REFERENCES p_ci_job_artifacts(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/doc/administration/geo/replication/troubleshooting/failover.md b/doc/administration/geo/replication/troubleshooting/failover.md index 8b51670a9ff..b5849fd161b 100644 --- a/doc/administration/geo/replication/troubleshooting/failover.md +++ b/doc/administration/geo/replication/troubleshooting/failover.md @@ -1,5 +1,5 @@ --- -redirect_to: (../../disaster_recovery/failover_troubleshooting.md) +redirect_to: '../../disaster_recovery/failover_troubleshooting.md' remove_date: '2025-08-02' --- @@ -8,4 +8,4 @@ This document was moved to [another location](../../disaster_recovery/failover_t - \ No newline at end of file + diff --git a/package.json b/package.json index 434199b0bc5..cb6ba1eac54 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,7 @@ "@gitlab/favicon-overlay": "2.0.0", "@gitlab/fonts": "^1.3.0", "@gitlab/svgs": "3.112.0", - "@gitlab/ui": "88.1.0", + "@gitlab/ui": "88.6.0", "@gitlab/web-ide": "^0.0.1-dev-20240731185426", "@mattiasbuelens/web-streams-adapter": "^0.1.0", "@rails/actioncable": "7.0.8-4", diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb index 19857227fd4..182b35c05e8 100755 --- a/scripts/review_apps/automated_cleanup.rb +++ b/scripts/review_apps/automated_cleanup.rb @@ -7,16 +7,14 @@ # See https://docs.gitlab.com/ee/development/pipelines/internals.html#using-the-gitlab-ruby-gem-in-the-canonical-project. require 'gitlab' require 'optparse' +require 'time' + require_relative File.expand_path('../../tooling/lib/tooling/helm3_client.rb', __dir__) require_relative File.expand_path('../../tooling/lib/tooling/kubernetes_client.rb', __dir__) module ReviewApps class AutomatedCleanup - DEPLOYMENTS_PER_PAGE = 100 - ENVIRONMENT_PREFIX = { - review_app: 'review/', - docs_review_app: 'review-docs/' - }.freeze + ENVIRONMENTS_PER_PAGE = 100 IGNORED_HELM_ERRORS = [ 'transport is closing', 'error upgrading connection', @@ -57,7 +55,6 @@ module ReviewApps @gitlab_token = gitlab_token @api_endpoint = api_endpoint @dry_run = options[:dry_run] - @environments_not_found_count = 0 end def gitlab @@ -80,81 +77,23 @@ module ReviewApps @kubernetes ||= Tooling::KubernetesClient.new end - def perform_gitlab_environment_cleanup!(days_for_delete:) + def perform_gitlab_environment_cleanup!(env_prefix:, days_for_delete:) puts "Dry-run mode." if dry_run - puts "Checking for Review Apps not updated in the last #{days_for_delete} days..." + puts "Checking for GitLab #{env_prefix} environments without any deployments in them, or deployed more than #{days_for_delete} days ago..." - checked_environments = [] delete_threshold = threshold_time(days: days_for_delete) - deployments_look_back_threshold = threshold_time(days: days_for_delete * 5) - releases_to_delete = [] + gitlab.environments(project_path, per_page: ENVIRONMENTS_PER_PAGE, sort: 'desc', search: env_prefix).auto_paginate do |environment| + next unless environment.name.start_with?(env_prefix) + # TODO: Find a way to reset those, so that we can properly delete them. + next if environment.state == 'stopping' # We cannot delete environments in stopping state - # Delete environments via deployments - gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment| - last_deploy = deployment.created_at - deployed_at = Time.parse(last_deploy) + deployments = gitlab.deployments(project_path, environment: environment.name, order_by: :created_at, sort: :desc) + next if deployments.count > 0 && Time.parse(deployments.first.created_at) > delete_threshold - break if deployed_at < deployments_look_back_threshold - - environment = deployment.environment - - next unless environment - next unless environment.name.start_with?(ENVIRONMENT_PREFIX[:review_app]) - next if checked_environments.include?(environment.slug) - - if deployed_at < delete_threshold - deleted_environment = delete_environment(environment, deployment) - - if deleted_environment - release = Tooling::Helm3Client::Release.new(name: environment.slug, namespace: environment.slug, revision: 1) - releases_to_delete << release - end - end - - checked_environments << environment.slug + stop_environment(environment) + delete_environment(environment) end - - delete_stopped_environments(environment_type: :review_app, checked_environments: checked_environments, last_updated_threshold: delete_threshold) do |environment| - releases_to_delete << Tooling::Helm3Client::Release.new(name: environment.slug, namespace: environment.slug, revision: 1, updated: environment.updated_at) - end - - delete_helm_releases(releases_to_delete) - end - - def perform_gitlab_docs_environment_cleanup!(days_for_stop:, days_for_delete:) - puts "Dry-run mode." if dry_run - puts "Checking for Docs Review Apps not updated in the last #{days_for_stop} days..." - - checked_environments = [] - stop_threshold = threshold_time(days: days_for_stop) - delete_threshold = threshold_time(days: days_for_delete) - deployments_look_back_threshold = threshold_time(days: days_for_delete * 5) - - # Delete environments via deployments - gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment| - last_deploy = deployment.created_at - deployed_at = Time.parse(last_deploy) - - break if deployed_at < deployments_look_back_threshold - - environment = deployment.environment - - next unless environment - next unless environment.name.start_with?(ENVIRONMENT_PREFIX[:docs_review_app]) - next if checked_environments.include?(environment.slug) - - if deployed_at < stop_threshold - environment_state = fetch_environment(environment)&.state - stop_environment(environment, deployment) if environment_state && environment_state != 'stopped' - end - - delete_environment(environment, deployment) if deployed_at < delete_threshold - - checked_environments << environment.slug - end - - delete_stopped_environments(environment_type: :docs_review_app, checked_environments: checked_environments, last_updated_threshold: delete_threshold) end def perform_helm_releases_cleanup!(days:) @@ -190,55 +129,26 @@ module ReviewApps attr_reader :api_endpoint, :dry_run, :gitlab_token, :project_path - def fetch_environment(environment) - gitlab.environment(project_path, environment.id) - rescue Errno::ETIMEDOUT => ex - puts "Failed to fetch '#{environment.name}' / '#{environment.slug}' (##{environment.id}):\n#{ex.message}" - nil - end + def stop_environment(environment) + return if environment.state == 'stopped' || environment.state == 'stopping' - def delete_environment(environment, deployment = nil) - release_date = deployment ? deployment.created_at : environment.updated_at - print_release_state(subject: 'Review app', release_name: environment.slug, release_date: release_date, action: 'deleting') - gitlab.delete_environment(project_path, environment.id) unless dry_run - - rescue Gitlab::Error::NotFound - puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) was not found: ignoring it" - @environments_not_found_count += 1 - - if @environments_not_found_count >= ENVIRONMENTS_NOT_FOUND_THRESHOLD - raise "At least #{ENVIRONMENTS_NOT_FOUND_THRESHOLD} environments were missing when we tried to delete them. Please investigate" - end - rescue Gitlab::Error::Forbidden - puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it" - rescue Gitlab::Error::InternalServerError - puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) 500 error: ignoring it" - end - - def stop_environment(environment, deployment) - print_release_state(subject: 'Review app', release_name: environment.slug, release_date: deployment.created_at, action: 'stopping') + print_release_state(subject: 'GitLab Environment', release_name: environment.slug, release_date: environment.created_at, action: 'stopping') gitlab.stop_environment(project_path, environment.id) unless dry_run - rescue Gitlab::Error::Forbidden - puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it" + puts "GitLab environment '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it" end - def delete_stopped_environments(environment_type:, checked_environments:, last_updated_threshold:) - gitlab.environments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc', states: 'stopped', search: ENVIRONMENT_PREFIX[environment_type]).auto_paginate do |environment| - next if skip_environment?(environment: environment, checked_environments: checked_environments, last_updated_threshold: last_updated_threshold, environment_type: environment_type) + def delete_environment(environment) + return if environment.state == 'stopping' - yield environment if delete_environment(environment) && block_given? - - checked_environments << environment.slug - end - end - - def skip_environment?(environment:, checked_environments:, last_updated_threshold:, environment_type:) - return true unless environment.name.start_with?(ENVIRONMENT_PREFIX[environment_type]) - return true if checked_environments.include?(environment.slug) - return true if Time.parse(environment.updated_at) > last_updated_threshold - - false + print_release_state(subject: 'GitLab environment', release_name: environment.slug, release_date: environment.created_at, action: 'deleting') + gitlab.delete_environment(project_path, environment.id) unless dry_run + rescue Gitlab::Error::NotFound + puts "GitLab environment '#{environment.name}' / '#{environment.slug}' (##{environment.id}) was not found: ignoring it" + rescue Gitlab::Error::Forbidden + puts "GitLab environment '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it" + rescue Gitlab::Error::InternalServerError + puts "GitLab environment '#{environment.name}' / '#{environment.slug}' (##{environment.id}) 500 error: ignoring it" end def helm_releases @@ -298,10 +208,6 @@ if $PROGRAM_NAME == __FILE__ options = ReviewApps::AutomatedCleanup.parse_args(ARGV) automated_cleanup = ReviewApps::AutomatedCleanup.new(options: options) - timed('Docs Review Apps cleanup') do - automated_cleanup.perform_gitlab_docs_environment_cleanup!(days_for_stop: 20, days_for_delete: 30) - end - puts timed('Helm releases cleanup') do @@ -310,8 +216,16 @@ if $PROGRAM_NAME == __FILE__ puts - timed('Review Apps cleanup') do - automated_cleanup.perform_gitlab_environment_cleanup!(days_for_delete: 7) + timed('Review Apps Environments cleanup') do + automated_cleanup.perform_gitlab_environment_cleanup!(env_prefix: 'review/', days_for_delete: 7) + end + + timed('Docs Review Apps environments cleanup') do + automated_cleanup.perform_gitlab_environment_cleanup!(env_prefix: 'review-docs/', days_for_delete: 30) + end + + timed('as-if-foss Environments cleanup') do + automated_cleanup.perform_gitlab_environment_cleanup!(env_prefix: 'as-if-foss/', days_for_delete: 30) end puts diff --git a/spec/frontend/vue_shared/components/list_selector/api_spec.js b/spec/frontend/vue_shared/components/list_selector/api_spec.js index 87fa740ce90..f85a6188e47 100644 --- a/spec/frontend/vue_shared/components/list_selector/api_spec.js +++ b/spec/frontend/vue_shared/components/list_selector/api_spec.js @@ -1,10 +1,13 @@ +import MockAdapter from 'axios-mock-adapter'; import Api from '~/api'; import { getProjects } from '~/rest_api'; import { ACCESS_LEVEL_REPORTER_INTEGER } from '~/access_level/constants'; import { getIdFromGraphQLId } from '~/graphql_shared/utils'; +import axios from '~/lib/utils/axios_utils'; import { fetchProjectGroups, fetchAllGroups, + fetchGroupsWithProjectAccess, fetchProjects, fetchUsers, } from '~/vue_shared/components/list_selector/api'; @@ -12,12 +15,13 @@ import { jest.mock('~/api'); jest.mock('~/rest_api'); jest.mock('~/graphql_shared/utils'); +let axiosMock; const mockProjectPath = 'group/project'; +const mockGroupSearch = 'group'; describe('List Selector Utils', () => { describe('fetchProjectGroups', () => { - const mockSearch = 'group'; const mockApiResponse = [ { id: 1, full_name: 'Group 1', name: 'group1' }, { id: 2, full_name: 'Group 2', name: 'group2' }, @@ -28,17 +32,17 @@ describe('List Selector Utils', () => { }); it('calls Api.projectGroups with correct parameters', async () => { - await fetchProjectGroups(mockProjectPath, mockSearch); + await fetchProjectGroups(mockProjectPath, mockGroupSearch); expect(Api.projectGroups).toHaveBeenCalledWith(mockProjectPath, { - search: mockSearch, + search: mockGroupSearch, with_shared: true, shared_min_access_level: ACCESS_LEVEL_REPORTER_INTEGER, }); }); it('returns formatted group data', async () => { - const result = await fetchProjectGroups(mockProjectPath, mockSearch); + const result = await fetchProjectGroups(mockProjectPath, mockGroupSearch); expect(result).toEqual([ { text: 'Group 1', value: 'group1', id: 1, fullName: 'Group 1', name: 'group1' }, @@ -51,7 +55,6 @@ describe('List Selector Utils', () => { const mockApollo = { query: jest.fn(), }; - const mockSearch = 'search-term'; const mockGraphQLResponse = { data: { groups: { @@ -69,16 +72,16 @@ describe('List Selector Utils', () => { }); it('calls apollo.query with correct parameters', async () => { - await fetchAllGroups(mockApollo, mockSearch); + await fetchAllGroups(mockApollo, mockGroupSearch); expect(mockApollo.query).toHaveBeenCalledWith({ query: expect.any(Object), - variables: { search: mockSearch }, + variables: { search: mockGroupSearch }, }); }); it('returns formatted group data', async () => { - const result = await fetchAllGroups(mockApollo, mockSearch); + const result = await fetchAllGroups(mockApollo, mockGroupSearch); expect(result).toEqual([ { @@ -101,6 +104,44 @@ describe('List Selector Utils', () => { }); }); + describe('fetchGroupsWithProjectAccess', () => { + const mockProjectId = 7; + const mockUrl = '/-/autocomplete/project_groups.json'; + + beforeEach(() => { + const mockAxiosResponse = [ + { id: 1, avatar_url: null, name: 'group1' }, + { id: 2, avatar_url: null, name: 'group2' }, + ]; + axiosMock = new MockAdapter(axios); + axiosMock.onGet(mockUrl).replyOnce(200, mockAxiosResponse); + }); + + afterEach(() => { + axiosMock.restore(); + }); + + it('calls axios.get with correct parameters', async () => { + await fetchGroupsWithProjectAccess(mockProjectId, mockGroupSearch); + + expect(axiosMock.history.get.length).toBe(1); + expect(axiosMock.history.get[0].params).toStrictEqual({ + project_id: mockProjectId, + with_project_access: true, + search: mockGroupSearch, + }); + }); + + it('returns formatted group data', async () => { + const result = await fetchGroupsWithProjectAccess(mockProjectId, mockGroupSearch); + + expect(result).toEqual([ + { text: 'group1', value: 'group1', id: 1, avatarUrl: null, name: 'group1' }, + { text: 'group2', value: 'group2', id: 2, avatarUrl: null, name: 'group2' }, + ]); + }); + }); + describe('fetchProjects', () => { beforeEach(() => { jest.clearAllMocks(); diff --git a/spec/frontend/vue_shared/components/list_selector/index_spec.js b/spec/frontend/vue_shared/components/list_selector/index_spec.js index b536e24ff95..99be940fc35 100644 --- a/spec/frontend/vue_shared/components/list_selector/index_spec.js +++ b/spec/frontend/vue_shared/components/list_selector/index_spec.js @@ -1,9 +1,11 @@ import Vue from 'vue'; import VueApollo from 'vue-apollo'; import { GlIcon, GlCollapsibleListbox, GlSearchBoxByType } from '@gitlab/ui'; +import MockAdapter from 'axios-mock-adapter'; import CrudComponent from '~/vue_shared/components/crud_component.vue'; import Api from '~/api'; import RestApi from '~/rest_api'; +import axios from '~/lib/utils/axios_utils'; import { createAlert } from '~/alert'; import { getIdFromGraphQLId } from '~/graphql_shared/utils'; import { mountExtended } from 'helpers/vue_test_utils_helper'; @@ -19,6 +21,7 @@ import { ACCESS_LEVEL_REPORTER_INTEGER } from '~/access_level/constants'; import { USERS_RESPONSE_MOCK, GROUPS_RESPONSE_MOCK } from './mock_data'; jest.mock('~/alert'); +jest.mock('~/api'); jest.mock('~/rest_api', () => ({ getProjects: jest.fn().mockResolvedValue({ data: [ @@ -32,6 +35,7 @@ Vue.use(VueApollo); describe('List Selector spec', () => { let wrapper; let fakeApollo; + let axiosMock; const USERS_MOCK_PROPS = { projectPath: 'some/project/path', @@ -185,55 +189,90 @@ describe('List Selector spec', () => { }); describe('searching', () => { - const searchResponse = GROUPS_RESPONSE_MOCK.data.groups.nodes.map((group) => ({ - ...group, - id: getIdFromGraphQLId(group.id), - })); + describe('for default all groups', () => { + const searchResponse = GROUPS_RESPONSE_MOCK.data.groups.nodes.map((group) => ({ + ...group, + id: getIdFromGraphQLId(group.id), + })); - const emitSearchInput = async () => { - findSearchBox().vm.$emit('input', search); - await waitForPromises(); - }; + const emitSearchInput = async () => { + findSearchBox().vm.$emit('input', search); + await waitForPromises(); + }; - beforeEach(async () => { - findNamespaceDropdown().vm.$emit('select', 'false'); - await emitSearchInput(); - }); + beforeEach(async () => { + findNamespaceDropdown().vm.$emit('select', 'false'); + await emitSearchInput(); + }); - it('calls query with correct variables when Search box receives an input', () => { - expect(groupsAutocompleteQuerySuccess).toHaveBeenCalledWith({ - search, + it('calls query with correct variables when Search box receives an input', () => { + expect(groupsAutocompleteQuerySuccess).toHaveBeenCalledWith({ + search, + }); + }); + + it('renders a dropdown for the search results', () => { + expect(findSearchResultsDropdown().props()).toMatchObject({ + items: searchResponse, + }); + }); + + it('renders a group component for each search result', () => { + expect(findAllGroupComponents().length).toBe(searchResponse.length); + }); + + it('emits an event when a search result is selected', () => { + const firstSearchResult = searchResponse[0]; + findSearchResultsDropdown().vm.$emit('select', firstSearchResult.name); + + expect(wrapper.emitted('select')).toEqual([ + [ + { + __typename: 'Group', + avatarUrl: null, + fullName: 'Flightjs', + id: 33, + name: 'Flightjs', + text: 'Flightjs', + value: 'Flightjs', + type: 'group', + }, + ], + ]); }); }); - it('renders a dropdown for the search results', () => { - expect(findSearchResultsDropdown().props()).toMatchObject({ - items: searchResponse, + describe('for groups with project access', () => { + const mockProjectId = 7; + const mockUrl = '/-/autocomplete/project_groups.json'; + const mockAxiosResponse = [ + { id: 1, avatar_url: null, name: 'group1' }, + { id: 2, avatar_url: null, name: 'group2' }, + ]; + axiosMock = new MockAdapter(axios); + + const emitSearchInput = async () => { + findSearchBox().vm.$emit('input', search); + await waitForPromises(); + }; + + beforeEach(async () => { + createComponent({ + ...GROUPS_MOCK_PROPS, + isGroupsWithProjectAccess: true, + projectId: mockProjectId, + }); + axiosMock.onGet(mockUrl).replyOnce(200, mockAxiosResponse); + await emitSearchInput(); }); - }); - it('renders a group component for each search result', () => { - expect(findAllGroupComponents().length).toBe(searchResponse.length); - }); - - it('emits an event when a search result is selected', () => { - const firstSearchResult = searchResponse[0]; - findSearchResultsDropdown().vm.$emit('select', firstSearchResult.name); - - expect(wrapper.emitted('select')).toEqual([ - [ - { - __typename: 'Group', - avatarUrl: null, - fullName: 'Flightjs', - id: 33, - name: 'Flightjs', - text: 'Flightjs', - value: 'Flightjs', - type: 'group', - }, - ], - ]); + it('calls query with correct variables when Search box receives an input', () => { + expect(axiosMock.history.get[0].params).toStrictEqual({ + project_id: mockProjectId, + with_project_access: true, + search, + }); + }); }); }); diff --git a/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb b/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb index 20a66e72e21..bb463741989 100644 --- a/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb +++ b/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb @@ -39,7 +39,8 @@ RSpec.describe 'cross-database foreign keys' do 'vulnerability_export_parts.vulnerability_export_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/473014 'vulnerability_flags.vulnerability_occurrence_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/473014 'vulnerability_finding_links.vulnerability_occurrence_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/473014 - 'dast_site_validations.dast_site_token_id' # https://gitlab.com/gitlab-org/gitlab/-/issues/474985 + 'dast_site_validations.dast_site_token_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/474985 + 'dast_sites.dast_site_validation_id' # https://gitlab.com/gitlab-org/gitlab/-/issues/476657 ] end diff --git a/spec/migrations/20240607105705_queue_backfill_dast_site_validations_project_id_spec.rb b/spec/migrations/20240607105705_queue_backfill_dast_site_validations_project_id_spec.rb index 45adea6c1ae..749caf69138 100644 --- a/spec/migrations/20240607105705_queue_backfill_dast_site_validations_project_id_spec.rb +++ b/spec/migrations/20240607105705_queue_backfill_dast_site_validations_project_id_spec.rb @@ -19,7 +19,7 @@ RSpec.describe QueueBackfillDastSiteValidationsProjectId, feature_category: :dyn interval: described_class::DELAY_INTERVAL, batch_size: described_class::BATCH_SIZE, sub_batch_size: described_class::SUB_BATCH_SIZE, - gitlab_schema: :gitlab_main_cell, + gitlab_schema: :gitlab_sec, job_arguments: [ :project_id, :dast_site_tokens, diff --git a/spec/models/ci/finished_pipeline_ch_sync_event_spec.rb b/spec/models/ci/finished_pipeline_ch_sync_event_spec.rb index 616b8f9221d..809be4ddc2b 100644 --- a/spec/models/ci/finished_pipeline_ch_sync_event_spec.rb +++ b/spec/models/ci/finished_pipeline_ch_sync_event_spec.rb @@ -115,7 +115,7 @@ RSpec.describe Ci::FinishedPipelineChSyncEvent, type: :model, feature_category: context 'when almost all the records are too old' do before do - travel(1.year - 2.minutes) + travel(30.days - 2.minutes) end it { is_expected.to eq(false) } @@ -123,7 +123,7 @@ RSpec.describe Ci::FinishedPipelineChSyncEvent, type: :model, feature_category: context 'when all the records are too old' do before do - travel(1.year) + travel(30.days) end it { is_expected.to eq(true) } diff --git a/spec/scripts/review_apps/automated_cleanup_spec.rb b/spec/scripts/review_apps/automated_cleanup_spec.rb index a8b8353d2ef..f3b74a9077a 100644 --- a/spec/scripts/review_apps/automated_cleanup_spec.rb +++ b/spec/scripts/review_apps/automated_cleanup_spec.rb @@ -23,6 +23,7 @@ RSpec.describe ReviewApps::AutomatedCleanup, feature_category: :tooling do let(:one_day_ago) { (now - (1 * 24 * 3600)) } let(:two_days_ago) { (now - (2 * 24 * 3600)) } let(:three_days_ago) { (now - (3 * 24 * 3600)) } + let(:ten_days_ago) { (now - (10 * 24 * 3600)) } before do allow(instance).to receive(:gitlab).and_return(gitlab_client) @@ -145,6 +146,161 @@ RSpec.describe ReviewApps::AutomatedCleanup, feature_category: :tooling do end end + describe '#perform_gitlab_environment_cleanup!' do + let(:env_prefix) { 'test-prefix/' } + let(:days_for_delete) { 2 } + let(:deployment_created_at) { three_days_ago.to_s } + let(:env_name) { "#{env_prefix}an-env-name" } + let(:env_state) { 'available' } + # rubocop:disable RSpec/VerifiedDoubles -- Internal API resource + let(:environments) do + [double('GitLab Environment', + id: env_name, + name: env_name, + slug: env_name, + state: env_state, + created_at: one_day_ago.to_s)] + end + + let(:deployments) do + [double('GitLab Deployment', created_at: deployment_created_at)] + end + + subject do + instance.perform_gitlab_environment_cleanup!( + env_prefix: env_prefix, + days_for_delete: days_for_delete + ) + end + + before do + allow(gitlab_client).to yield_environments(:environments, environments) + allow(gitlab_client).to receive(:deployments).and_return(deployments) + + # Silence outputs to stdout + allow(instance).to receive(:puts) + end + + def yield_environments(api_method, environments) + messages = receive_message_chain(api_method, :auto_paginate) + + environments.inject(messages) do |stub, environment| + stub.and_yield(environment) + end + end + + # rubocop:enable RSpec/VerifiedDoubles -- Internal API resource + + context 'when the environment is not for a review-app' do + let(:env_name) { 'not-for-a-review-app' } + + it 'does not stop the environment' do + expect(gitlab_client).not_to receive(:stop_environment) + + subject + end + + it 'does not delete the environment' do + expect(gitlab_client).not_to receive(:delete_environment) + + subject + end + end + + context 'when the environment is for a review-app' do + context 'when there are no deployments' do + let(:deployments) { [] } + + context 'when the environment is already stopped' do + let(:env_state) { 'stopped' } + + it 'does not stop the environment' do + expect(gitlab_client).not_to receive(:stop_environment) + allow(gitlab_client).to receive(:delete_environment) + + subject + end + + it 'deletes the environment' do + expect(gitlab_client).to receive(:delete_environment) + + subject + end + end + + context 'when the environment is not stopped' do + let(:env_state) { 'available' } + + it 'stops the environment' do + expect(gitlab_client).to receive(:stop_environment) + allow(gitlab_client).to receive(:delete_environment) + + subject + end + + it 'deletes the environment' do + allow(gitlab_client).to receive(:stop_environment) + expect(gitlab_client).to receive(:delete_environment) + + subject + end + end + end + + context 'when the environment state is stopping' do + let(:env_state) { 'stopping' } + + it 'does not stop the environment' do + expect(gitlab_client).not_to receive(:stop_environment) + + subject + end + + it 'does not delete the environment' do + expect(gitlab_client).not_to receive(:delete_environment) + + subject + end + end + + context 'when there are deployments' do + context 'when the latest deployment happened later than the days_for_delete argument' do + let(:deployment_created_at) { one_day_ago.to_s } + + it 'does not stop the environment' do + expect(gitlab_client).not_to receive(:stop_environment) + + subject + end + + it 'does not delete the environment' do + expect(gitlab_client).not_to receive(:delete_environment) + + subject + end + end + + context 'when the latest deployment happened earlier than the days_for_delete argument' do + let(:deployment_created_at) { three_days_ago.to_s } + + it 'stops the environment' do + expect(gitlab_client).to receive(:stop_environment) + allow(gitlab_client).to receive(:delete_environment) + + subject + end + + it 'deletes the environment' do + allow(gitlab_client).to receive(:stop_environment) + expect(gitlab_client).to receive(:delete_environment) + + subject + end + end + end + end + end + describe '#perform_helm_releases_cleanup!' do subject { instance.perform_helm_releases_cleanup!(days: days) } diff --git a/yarn.lock b/yarn.lock index 111d9933175..d41a3ca1eb3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1350,10 +1350,10 @@ resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.112.0.tgz#9789d6766de222762594c8512bc3b32b619dfe6b" integrity sha512-lemHTryUoeWcOPd1V8B4Iag6uXU0DvEqfKibPS6PhXAsPdMTCObMb7RPEeVvGp0KI/05fWYJ5EpEBDm5nv0EIw== -"@gitlab/ui@88.1.0": - version "88.1.0" - resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-88.1.0.tgz#67eac5bffbfa5acc552e00cf3d8b9dee69d63378" - integrity sha512-y/9MJfAzJlvBykgzPket6/bYxh00B3LfwabTPdBar6Yh8KrIKlM2hD+fZQt7SeZcS31ILyM2kVgVlV4AK9M6Ag== +"@gitlab/ui@88.6.0": + version "88.6.0" + resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-88.6.0.tgz#3322b44e2095f67fc2fab7a77fec3d3c28692f57" + integrity sha512-h36nmo78aTthjc6jxxRPDQIi9Oaw4RIU1GYkLrlr2Cs/Ah06QvXFH52hp2lQd/5VCZau8jHIXDV97NKhS9E2Xw== dependencies: "@floating-ui/dom" "1.4.3" echarts "^5.3.2"