diff --git a/app/assets/javascripts/packages_and_registries/container_registry/explorer/components/details_page/tags_list.vue b/app/assets/javascripts/packages_and_registries/container_registry/explorer/components/details_page/tags_list.vue index aa654a0e3c2..f79f7f6d372 100644 --- a/app/assets/javascripts/packages_and_registries/container_registry/explorer/components/details_page/tags_list.vue +++ b/app/assets/javascripts/packages_and_registries/container_registry/explorer/components/details_page/tags_list.vue @@ -19,6 +19,7 @@ import { REMOVE_TAGS_BUTTON_TITLE, TAGS_LIST_TITLE, GRAPHQL_PAGE_SIZE, + GRAPHQL_PAGE_SIZE_METADATA_ENABLED, FETCH_IMAGES_LIST_ERROR_MESSAGE, NAME_SORT_FIELD, PUBLISHED_SORT_FIELD, @@ -116,10 +117,15 @@ export default { tagsPageInfo() { return this.containerRepository?.tags?.pageInfo; }, + pageSize() { + return this.config.isMetadataDatabaseEnabled + ? GRAPHQL_PAGE_SIZE_METADATA_ENABLED + : GRAPHQL_PAGE_SIZE; + }, queryVariables() { return { id: joinPaths(this.config.gidPrefix, `${this.id}`), - first: GRAPHQL_PAGE_SIZE, + first: this.pageSize, name: this.filters?.name, sort: this.sort, referrers: this.glFeatures.showContainerRegistryTagSignatures, @@ -196,13 +202,13 @@ export default { } }, fetchNextPage() { - this.pageParams = getNextPageParams(this.tagsPageInfo?.endCursor); + this.pageParams = getNextPageParams(this.tagsPageInfo?.endCursor, this.pageSize); }, fetchPreviousPage() { - this.pageParams = getPreviousPageParams(this.tagsPageInfo?.startCursor); + this.pageParams = getPreviousPageParams(this.tagsPageInfo?.startCursor, this.pageSize); }, handleSearchUpdate({ sort, filters, pageInfo }) { - this.pageParams = getPageParams(pageInfo); + this.pageParams = getPageParams(pageInfo, this.pageSize); this.sort = sort; // This takes in account the fact that we will be adding more filters types diff --git a/app/assets/javascripts/packages_and_registries/container_registry/explorer/constants/list.js b/app/assets/javascripts/packages_and_registries/container_registry/explorer/constants/list.js index cbab034d9d4..4698049dd6c 100644 --- a/app/assets/javascripts/packages_and_registries/container_registry/explorer/constants/list.js +++ b/app/assets/javascripts/packages_and_registries/container_registry/explorer/constants/list.js @@ -50,6 +50,7 @@ export const TRACKING_ACTION_CLICK_SHOW_FULL_PATH = 'click_show_full_path'; export const IMAGE_DELETE_SCHEDULED_STATUS = 'DELETE_SCHEDULED'; export const IMAGE_MIGRATING_STATE = 'importing'; export const GRAPHQL_PAGE_SIZE = 10; +export const GRAPHQL_PAGE_SIZE_METADATA_ENABLED = 20; export const SORT_FIELDS = [ { orderBy: 'UPDATED', label: __('Updated') }, diff --git a/app/assets/javascripts/packages_and_registries/container_registry/explorer/pages/list.vue b/app/assets/javascripts/packages_and_registries/container_registry/explorer/pages/list.vue index 2ececeb606e..e58a7920c91 100644 --- a/app/assets/javascripts/packages_and_registries/container_registry/explorer/pages/list.vue +++ b/app/assets/javascripts/packages_and_registries/container_registry/explorer/pages/list.vue @@ -29,6 +29,7 @@ import { EMPTY_RESULT_TITLE, EMPTY_RESULT_MESSAGE, GRAPHQL_PAGE_SIZE, + GRAPHQL_PAGE_SIZE_METADATA_ENABLED, FETCH_IMAGES_LIST_ERROR_MESSAGE, SORT_FIELDS, SETTINGS_TEXT, @@ -159,13 +160,18 @@ export default { graphqlResource() { return this.config.isGroupPage ? WORKSPACE_GROUP : WORKSPACE_PROJECT; }, + pageSize() { + return this.config.isMetadataDatabaseEnabled + ? GRAPHQL_PAGE_SIZE_METADATA_ENABLED + : GRAPHQL_PAGE_SIZE; + }, queryVariables() { return { name: this.name, sort: this.sorting, fullPath: this.config.isGroupPage ? this.config.groupPath : this.config.projectPath, isGroupPage: this.config.isGroupPage, - first: GRAPHQL_PAGE_SIZE, + first: this.pageSize, ...this.pageParams, }; }, @@ -203,17 +209,17 @@ export default { this.itemToDelete = {}; }, fetchNextPage() { - this.pageParams = getNextPageParams(this.pageInfo?.endCursor); + this.pageParams = getNextPageParams(this.pageInfo?.endCursor, this.pageSize); }, fetchPreviousPage() { - this.pageParams = getPreviousPageParams(this.pageInfo?.startCursor); + this.pageParams = getPreviousPageParams(this.pageInfo?.startCursor, this.pageSize); }, startDelete() { this.track('confirm_delete'); this.mutationLoading = true; }, handleSearchUpdate({ sort, filters, pageInfo }) { - this.pageParams = getPageParams(pageInfo); + this.pageParams = getPageParams(pageInfo, this.pageSize); this.sorting = sort; const search = filters.find((i) => i.type === FILTERED_SEARCH_TERM); diff --git a/app/assets/javascripts/packages_and_registries/container_registry/explorer/utils.js b/app/assets/javascripts/packages_and_registries/container_registry/explorer/utils.js index 7ed4ff52b06..87c28e6e9b2 100644 --- a/app/assets/javascripts/packages_and_registries/container_registry/explorer/utils.js +++ b/app/assets/javascripts/packages_and_registries/container_registry/explorer/utils.js @@ -12,24 +12,24 @@ export const timeTilRun = (time) => { return approximateDuration(difference / 1000); }; -export const getNextPageParams = (cursor) => ({ +export const getNextPageParams = (cursor, pageSize = GRAPHQL_PAGE_SIZE) => ({ after: cursor, - first: GRAPHQL_PAGE_SIZE, + first: pageSize, }); -export const getPreviousPageParams = (cursor) => ({ +export const getPreviousPageParams = (cursor, pageSize = GRAPHQL_PAGE_SIZE) => ({ first: null, before: cursor, - last: GRAPHQL_PAGE_SIZE, + last: pageSize, }); -export const getPageParams = (pageInfo = {}) => { +export const getPageParams = (pageInfo = {}, pageSize = GRAPHQL_PAGE_SIZE) => { if (pageInfo.before) { - return getPreviousPageParams(pageInfo.before); + return getPreviousPageParams(pageInfo.before, pageSize); } if (pageInfo.after) { - return getNextPageParams(pageInfo.after); + return getNextPageParams(pageInfo.after, pageSize); } return {}; diff --git a/app/views/projects/empty.html.haml b/app/views/projects/empty.html.haml index 3c37cbb490f..e7d669156c0 100644 --- a/app/views/projects/empty.html.haml +++ b/app/views/projects/empty.html.haml @@ -36,13 +36,13 @@ = _('You can also upload existing files from your computer using the instructions below.') .git-empty.js-git-empty %h5= _('Git global setup') - %pre.code.js-syntax-highlight + %pre.js-syntax-highlight :preserve git config --global user.name "#{h git_user_name}" git config --global user.email "#{h git_user_email}" %h5= _('Create a new repository') - %pre.code.js-syntax-highlight + %pre.js-syntax-highlight :preserve git clone #{ content_tag(:span, default_url_to_repo, class: 'js-clone')} cd #{h @project.path} @@ -55,7 +55,7 @@ git push --set-upstream origin #{h escaped_default_branch_name } %h5= _('Push an existing folder') - %pre.code.js-syntax-highlight + %pre.js-syntax-highlight :preserve cd existing_folder git init --initial-branch=#{h escaped_default_branch_name} @@ -67,7 +67,7 @@ git push --set-upstream origin #{h escaped_default_branch_name } %h5= _('Push an existing Git repository') - %pre.code.js-syntax-highlight + %pre.js-syntax-highlight :preserve cd existing_repo git remote rename origin old-origin diff --git a/db/docs/batched_background_migrations/backfill_compliance_framework_security_policies_namespace_id.yml b/db/docs/batched_background_migrations/backfill_compliance_framework_security_policies_namespace_id.yml new file mode 100644 index 00000000000..9e8ebd3bac4 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_compliance_framework_security_policies_namespace_id.yml @@ -0,0 +1,9 @@ +--- +migration_job_name: BackfillComplianceFrameworkSecurityPoliciesNamespaceId +description: Backfills sharding key `compliance_framework_security_policies.namespace_id` from `security_orchestration_policy_configurations`. +feature_category: security_policy_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/160211 +milestone: '17.3' +queued_migration_version: 20240722095920 +finalize_after: '2024-08-22' +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/batched_background_migrations/backfill_compliance_framework_security_policies_project_id.yml b/db/docs/batched_background_migrations/backfill_compliance_framework_security_policies_project_id.yml new file mode 100644 index 00000000000..9e59b1044ce --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_compliance_framework_security_policies_project_id.yml @@ -0,0 +1,9 @@ +--- +migration_job_name: BackfillComplianceFrameworkSecurityPoliciesProjectId +description: Backfills sharding key `compliance_framework_security_policies.project_id` from `security_orchestration_policy_configurations`. +feature_category: security_policy_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/160211 +milestone: '17.3' +queued_migration_version: 20240722095915 +finalize_after: '2024-08-22' +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/batched_background_migrations/deduplicate_lfs_objects_projects.yml b/db/docs/batched_background_migrations/deduplicate_lfs_objects_projects.yml new file mode 100644 index 00000000000..1db793adeb1 --- /dev/null +++ b/db/docs/batched_background_migrations/deduplicate_lfs_objects_projects.yml @@ -0,0 +1,11 @@ +--- +migration_job_name: DeduplicateLfsObjectsProjects +description: >- + This migration deduplicates lfs_objects_projects by lfs_object_id, project_id and repository_type. + After the migration is finalized, we need to add a unique index on all three columns to ensure + data consistency since the unique validation already exists at the model level. +feature_category: source_code_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/154323 +milestone: '17.3' +queued_migration_version: 20240808125149 +finalize_after: '2024-08-02' # required upgrade stop due date diff --git a/db/docs/compliance_framework_security_policies.yml b/db/docs/compliance_framework_security_policies.yml index 7d7a391580d..b8a42f3af85 100644 --- a/db/docs/compliance_framework_security_policies.yml +++ b/db/docs/compliance_framework_security_policies.yml @@ -27,3 +27,6 @@ desired_sharding_key: table: security_orchestration_policy_configurations sharding_key: namespace_id belongs_to: policy_configuration +desired_sharding_key_migration_job_name: +- BackfillComplianceFrameworkSecurityPoliciesProjectId +- BackfillComplianceFrameworkSecurityPoliciesNamespaceId diff --git a/db/migrate/20240722095911_add_project_id_to_compliance_framework_security_policies.rb b/db/migrate/20240722095911_add_project_id_to_compliance_framework_security_policies.rb new file mode 100644 index 00000000000..22c705dc0b5 --- /dev/null +++ b/db/migrate/20240722095911_add_project_id_to_compliance_framework_security_policies.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddProjectIdToComplianceFrameworkSecurityPolicies < Gitlab::Database::Migration[2.2] + milestone '17.3' + + def change + add_column :compliance_framework_security_policies, :project_id, :bigint + end +end diff --git a/db/migrate/20240722095916_add_namespace_id_to_compliance_framework_security_policies.rb b/db/migrate/20240722095916_add_namespace_id_to_compliance_framework_security_policies.rb new file mode 100644 index 00000000000..28bf19e88d1 --- /dev/null +++ b/db/migrate/20240722095916_add_namespace_id_to_compliance_framework_security_policies.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddNamespaceIdToComplianceFrameworkSecurityPolicies < Gitlab::Database::Migration[2.2] + milestone '17.3' + + def change + add_column :compliance_framework_security_policies, :namespace_id, :bigint + end +end diff --git a/db/migrate/20240806161731_add_index_to_member_roles_on_permissions.rb b/db/migrate/20240806161731_add_index_to_member_roles_on_permissions.rb new file mode 100644 index 00000000000..19381653458 --- /dev/null +++ b/db/migrate/20240806161731_add_index_to_member_roles_on_permissions.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddIndexToMemberRolesOnPermissions < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + milestone '17.3' + + INDEX_NAME = 'index_member_roles_on_permissions' + + def up + add_concurrent_index :member_roles, :permissions, name: INDEX_NAME, using: :gin + end + + def down + remove_concurrent_index_by_name :member_roles, INDEX_NAME + end +end diff --git a/db/migrate/20240807103912_add_vulnerability_count_to_project_statistics_table.rb b/db/migrate/20240807103912_add_vulnerability_count_to_project_statistics_table.rb new file mode 100644 index 00000000000..cf0b4b5b082 --- /dev/null +++ b/db/migrate/20240807103912_add_vulnerability_count_to_project_statistics_table.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddVulnerabilityCountToProjectStatisticsTable < Gitlab::Database::Migration[2.2] + milestone '17.4' + + def change + add_column :project_statistics, :vulnerability_count, :integer, default: 0, null: false + end +end diff --git a/db/post_migrate/20240722095912_index_compliance_framework_security_policies_on_project_id.rb b/db/post_migrate/20240722095912_index_compliance_framework_security_policies_on_project_id.rb new file mode 100644 index 00000000000..7453ea09e9c --- /dev/null +++ b/db/post_migrate/20240722095912_index_compliance_framework_security_policies_on_project_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexComplianceFrameworkSecurityPoliciesOnProjectId < Gitlab::Database::Migration[2.2] + milestone '17.3' + disable_ddl_transaction! + + INDEX_NAME = 'index_compliance_framework_security_policies_on_project_id' + + def up + add_concurrent_index :compliance_framework_security_policies, :project_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :compliance_framework_security_policies, INDEX_NAME + end +end diff --git a/db/post_migrate/20240722095913_add_compliance_framework_security_policies_project_id_fk.rb b/db/post_migrate/20240722095913_add_compliance_framework_security_policies_project_id_fk.rb new file mode 100644 index 00000000000..7f7d720c554 --- /dev/null +++ b/db/post_migrate/20240722095913_add_compliance_framework_security_policies_project_id_fk.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddComplianceFrameworkSecurityPoliciesProjectIdFk < Gitlab::Database::Migration[2.2] + milestone '17.3' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :compliance_framework_security_policies, :projects, column: :project_id, + on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :compliance_framework_security_policies, column: :project_id + end + end +end diff --git a/db/post_migrate/20240722095914_add_compliance_framework_security_policies_project_id_trigger.rb b/db/post_migrate/20240722095914_add_compliance_framework_security_policies_project_id_trigger.rb new file mode 100644 index 00000000000..a8cb8b8dca0 --- /dev/null +++ b/db/post_migrate/20240722095914_add_compliance_framework_security_policies_project_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddComplianceFrameworkSecurityPoliciesProjectIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.3' + + def up + install_sharding_key_assignment_trigger( + table: :compliance_framework_security_policies, + sharding_key: :project_id, + parent_table: :security_orchestration_policy_configurations, + parent_sharding_key: :project_id, + foreign_key: :policy_configuration_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :compliance_framework_security_policies, + sharding_key: :project_id, + parent_table: :security_orchestration_policy_configurations, + parent_sharding_key: :project_id, + foreign_key: :policy_configuration_id + ) + end +end diff --git a/db/post_migrate/20240722095915_queue_backfill_compliance_framework_security_policies_project_id.rb b/db/post_migrate/20240722095915_queue_backfill_compliance_framework_security_policies_project_id.rb new file mode 100644 index 00000000000..39201fee741 --- /dev/null +++ b/db/post_migrate/20240722095915_queue_backfill_compliance_framework_security_policies_project_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillComplianceFrameworkSecurityPoliciesProjectId < Gitlab::Database::Migration[2.2] + milestone '17.3' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillComplianceFrameworkSecurityPoliciesProjectId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :compliance_framework_security_policies, + :id, + :project_id, + :security_orchestration_policy_configurations, + :project_id, + :policy_configuration_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :compliance_framework_security_policies, + :id, + [ + :project_id, + :security_orchestration_policy_configurations, + :project_id, + :policy_configuration_id + ] + ) + end +end diff --git a/db/post_migrate/20240722095917_index_compliance_framework_security_policies_on_namespace_id.rb b/db/post_migrate/20240722095917_index_compliance_framework_security_policies_on_namespace_id.rb new file mode 100644 index 00000000000..3e6621ad7d1 --- /dev/null +++ b/db/post_migrate/20240722095917_index_compliance_framework_security_policies_on_namespace_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexComplianceFrameworkSecurityPoliciesOnNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.3' + disable_ddl_transaction! + + INDEX_NAME = 'index_compliance_framework_security_policies_on_namespace_id' + + def up + add_concurrent_index :compliance_framework_security_policies, :namespace_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :compliance_framework_security_policies, INDEX_NAME + end +end diff --git a/db/post_migrate/20240722095918_add_compliance_framework_security_policies_namespace_id_fk.rb b/db/post_migrate/20240722095918_add_compliance_framework_security_policies_namespace_id_fk.rb new file mode 100644 index 00000000000..5cec8b1988e --- /dev/null +++ b/db/post_migrate/20240722095918_add_compliance_framework_security_policies_namespace_id_fk.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddComplianceFrameworkSecurityPoliciesNamespaceIdFk < Gitlab::Database::Migration[2.2] + milestone '17.3' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :compliance_framework_security_policies, :namespaces, column: :namespace_id, + on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :compliance_framework_security_policies, column: :namespace_id + end + end +end diff --git a/db/post_migrate/20240722095919_add_compliance_framework_security_policies_namespace_id_trigger.rb b/db/post_migrate/20240722095919_add_compliance_framework_security_policies_namespace_id_trigger.rb new file mode 100644 index 00000000000..69d43fb66fb --- /dev/null +++ b/db/post_migrate/20240722095919_add_compliance_framework_security_policies_namespace_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddComplianceFrameworkSecurityPoliciesNamespaceIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.3' + + def up + install_sharding_key_assignment_trigger( + table: :compliance_framework_security_policies, + sharding_key: :namespace_id, + parent_table: :security_orchestration_policy_configurations, + parent_sharding_key: :namespace_id, + foreign_key: :policy_configuration_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :compliance_framework_security_policies, + sharding_key: :namespace_id, + parent_table: :security_orchestration_policy_configurations, + parent_sharding_key: :namespace_id, + foreign_key: :policy_configuration_id + ) + end +end diff --git a/db/post_migrate/20240722095920_queue_backfill_compliance_framework_security_policies_namespace_id.rb b/db/post_migrate/20240722095920_queue_backfill_compliance_framework_security_policies_namespace_id.rb new file mode 100644 index 00000000000..4269e0e791b --- /dev/null +++ b/db/post_migrate/20240722095920_queue_backfill_compliance_framework_security_policies_namespace_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillComplianceFrameworkSecurityPoliciesNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.3' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillComplianceFrameworkSecurityPoliciesNamespaceId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :compliance_framework_security_policies, + :id, + :namespace_id, + :security_orchestration_policy_configurations, + :namespace_id, + :policy_configuration_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :compliance_framework_security_policies, + :id, + [ + :namespace_id, + :security_orchestration_policy_configurations, + :namespace_id, + :policy_configuration_id + ] + ) + end +end diff --git a/db/post_migrate/20240808125149_queue_deduplicate_lfs_objects_projects.rb b/db/post_migrate/20240808125149_queue_deduplicate_lfs_objects_projects.rb new file mode 100644 index 00000000000..399563a843b --- /dev/null +++ b/db/post_migrate/20240808125149_queue_deduplicate_lfs_objects_projects.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class QueueDeduplicateLfsObjectsProjects < Gitlab::Database::Migration[2.2] + milestone '17.3' + + MIGRATION = 'DeduplicateLfsObjectsProjects' + TABLE_NAME = :lfs_objects_projects + DELAY_INTERVAL = 100 + BATCH_SIZE = 10_000 + SUB_BATCH_SIZE = 2_500 + + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :lfs_objects_projects, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :lfs_objects_projects, :id, []) + end +end diff --git a/db/schema_migrations/20240722095911 b/db/schema_migrations/20240722095911 new file mode 100644 index 00000000000..91d134eebdd --- /dev/null +++ b/db/schema_migrations/20240722095911 @@ -0,0 +1 @@ +6a352cd04798df6f147e70e05771e3047bfa52410fb4d08213414d21323d8304 \ No newline at end of file diff --git a/db/schema_migrations/20240722095912 b/db/schema_migrations/20240722095912 new file mode 100644 index 00000000000..06989243410 --- /dev/null +++ b/db/schema_migrations/20240722095912 @@ -0,0 +1 @@ +fe4c99013d813740cd59b0ed3a244d198233b2c1931b727ed751ec9d5ad373f6 \ No newline at end of file diff --git a/db/schema_migrations/20240722095913 b/db/schema_migrations/20240722095913 new file mode 100644 index 00000000000..ded6e6202cf --- /dev/null +++ b/db/schema_migrations/20240722095913 @@ -0,0 +1 @@ +7578b71069ea28c4d392ddaad249bd349ec3593d0a3ea23ed18958611d18e5bc \ No newline at end of file diff --git a/db/schema_migrations/20240722095914 b/db/schema_migrations/20240722095914 new file mode 100644 index 00000000000..4d6ac52128a --- /dev/null +++ b/db/schema_migrations/20240722095914 @@ -0,0 +1 @@ +5b49ae910f52c8bf975f77f8103f7d688c345cc8ee31e74d6417f02e1a819ef2 \ No newline at end of file diff --git a/db/schema_migrations/20240722095915 b/db/schema_migrations/20240722095915 new file mode 100644 index 00000000000..9dd2f77ae2b --- /dev/null +++ b/db/schema_migrations/20240722095915 @@ -0,0 +1 @@ +2c4b6b13a8ae5638792d3756ac9013f3ea822fac4421b887a7176e14e9a48277 \ No newline at end of file diff --git a/db/schema_migrations/20240722095916 b/db/schema_migrations/20240722095916 new file mode 100644 index 00000000000..d61ca823f7b --- /dev/null +++ b/db/schema_migrations/20240722095916 @@ -0,0 +1 @@ +111c47eb2171b12834c880b3dbeb133ba8dc9407da105a4973f791cb4aba9f9a \ No newline at end of file diff --git a/db/schema_migrations/20240722095917 b/db/schema_migrations/20240722095917 new file mode 100644 index 00000000000..c4104a95806 --- /dev/null +++ b/db/schema_migrations/20240722095917 @@ -0,0 +1 @@ +70aa8bbf10488c9e5177abcdb7b0565343758b7f47377cacc0d1db167759574d \ No newline at end of file diff --git a/db/schema_migrations/20240722095918 b/db/schema_migrations/20240722095918 new file mode 100644 index 00000000000..3bda1b55da1 --- /dev/null +++ b/db/schema_migrations/20240722095918 @@ -0,0 +1 @@ +3d76693b6c6b553c2bd681993a1d6e06b7d23d73050fa80294ecf660f2a9d10b \ No newline at end of file diff --git a/db/schema_migrations/20240722095919 b/db/schema_migrations/20240722095919 new file mode 100644 index 00000000000..9788256f47f --- /dev/null +++ b/db/schema_migrations/20240722095919 @@ -0,0 +1 @@ +28409fa04247197472cc18f20ba35c6323cd13c97acd7541a4d1873b71b5b43e \ No newline at end of file diff --git a/db/schema_migrations/20240722095920 b/db/schema_migrations/20240722095920 new file mode 100644 index 00000000000..e50a30a3ddc --- /dev/null +++ b/db/schema_migrations/20240722095920 @@ -0,0 +1 @@ +9f6bfee305d6b1239edbc7e2fdaa1563dcca1c465141582a714a6e2a31333baf \ No newline at end of file diff --git a/db/schema_migrations/20240806161731 b/db/schema_migrations/20240806161731 new file mode 100644 index 00000000000..b64ed385986 --- /dev/null +++ b/db/schema_migrations/20240806161731 @@ -0,0 +1 @@ +b1fbfb48e36e0fce50c27fec63cf83ce9c78a317a426733ade3e6863291e9d6a \ No newline at end of file diff --git a/db/schema_migrations/20240807103912 b/db/schema_migrations/20240807103912 new file mode 100644 index 00000000000..c2d984fcd08 --- /dev/null +++ b/db/schema_migrations/20240807103912 @@ -0,0 +1 @@ +c51b3cb6d6e9f0cdb2169322bb94641a2f8fabb2f4343b44af2905ceb2a4bcee \ No newline at end of file diff --git a/db/schema_migrations/20240808125149 b/db/schema_migrations/20240808125149 new file mode 100644 index 00000000000..98be0278225 --- /dev/null +++ b/db/schema_migrations/20240808125149 @@ -0,0 +1 @@ +b484b543db35e018c2a9bb9bfa3d0ff65f8f0ad6874509841660275ccef6c855 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 524097ded5b..b3e6b7a9892 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1330,6 +1330,22 @@ RETURN NEW; END $$; +CREATE FUNCTION trigger_6bf50b363152() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."project_id" IS NULL THEN + SELECT "project_id" + INTO NEW."project_id" + FROM "security_orchestration_policy_configurations" + WHERE "security_orchestration_policy_configurations"."id" = NEW."policy_configuration_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_6c38ba395cc1() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -1362,6 +1378,22 @@ RETURN NEW; END $$; +CREATE FUNCTION trigger_70d3f0bba1de() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."namespace_id" IS NULL THEN + SELECT "namespace_id" + INTO NEW."namespace_id" + FROM "security_orchestration_policy_configurations" + WHERE "security_orchestration_policy_configurations"."id" = NEW."policy_configuration_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_77d9fbad5b12() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -8994,7 +9026,9 @@ CREATE TABLE compliance_framework_security_policies ( policy_configuration_id bigint NOT NULL, created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, - policy_index smallint NOT NULL + policy_index smallint NOT NULL, + project_id bigint, + namespace_id bigint ); CREATE SEQUENCE compliance_framework_security_policies_id_seq @@ -16395,7 +16429,8 @@ CREATE TABLE project_statistics ( container_registry_size bigint DEFAULT 0 NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, updated_at timestamp with time zone DEFAULT now() NOT NULL, - root_namespace_id bigint + root_namespace_id bigint, + vulnerability_count integer DEFAULT 0 NOT NULL ); CREATE SEQUENCE project_statistics_id_seq @@ -27249,6 +27284,10 @@ CREATE UNIQUE INDEX index_commit_user_mentions_on_note_id ON commit_user_mention CREATE INDEX index_compliance_checks_on_namespace_id ON compliance_checks USING btree (namespace_id); +CREATE INDEX index_compliance_framework_security_policies_on_namespace_id ON compliance_framework_security_policies USING btree (namespace_id); + +CREATE INDEX index_compliance_framework_security_policies_on_project_id ON compliance_framework_security_policies USING btree (project_id); + CREATE INDEX index_compliance_frameworks_id_where_frameworks_not_null ON compliance_management_frameworks USING btree (id) WHERE (pipeline_configuration_full_path IS NOT NULL); CREATE INDEX index_compliance_management_frameworks_on_name_trigram ON compliance_management_frameworks USING gin (name gin_trgm_ops); @@ -28215,6 +28254,8 @@ CREATE UNIQUE INDEX index_member_roles_on_namespace_id_name_unique ON member_rol CREATE INDEX index_member_roles_on_occupies_seat ON member_roles USING btree (occupies_seat); +CREATE INDEX index_member_roles_on_permissions ON member_roles USING gin (permissions); + CREATE INDEX index_members_on_access_level ON members USING btree (access_level); CREATE INDEX index_members_on_expires_at ON members USING btree (expires_at); @@ -32205,10 +32246,14 @@ CREATE TRIGGER trigger_664594a3d0a7 BEFORE INSERT OR UPDATE ON merge_request_use CREATE TRIGGER trigger_68435a54ee2b BEFORE INSERT OR UPDATE ON packages_debian_project_architectures FOR EACH ROW EXECUTE FUNCTION trigger_68435a54ee2b(); +CREATE TRIGGER trigger_6bf50b363152 BEFORE INSERT OR UPDATE ON compliance_framework_security_policies FOR EACH ROW EXECUTE FUNCTION trigger_6bf50b363152(); + CREATE TRIGGER trigger_6c38ba395cc1 BEFORE INSERT OR UPDATE ON error_tracking_error_events FOR EACH ROW EXECUTE FUNCTION trigger_6c38ba395cc1(); CREATE TRIGGER trigger_6cdea9559242 BEFORE INSERT OR UPDATE ON issue_links FOR EACH ROW EXECUTE FUNCTION trigger_6cdea9559242(); +CREATE TRIGGER trigger_70d3f0bba1de BEFORE INSERT OR UPDATE ON compliance_framework_security_policies FOR EACH ROW EXECUTE FUNCTION trigger_70d3f0bba1de(); + CREATE TRIGGER trigger_77d9fbad5b12 BEFORE INSERT OR UPDATE ON packages_debian_project_distribution_keys FOR EACH ROW EXECUTE FUNCTION trigger_77d9fbad5b12(); CREATE TRIGGER trigger_7a8b08eed782 BEFORE INSERT OR UPDATE ON boards_epic_board_positions FOR EACH ROW EXECUTE FUNCTION trigger_7a8b08eed782(); @@ -32722,6 +32767,9 @@ ALTER TABLE ONLY release_links ALTER TABLE ONLY bulk_import_export_uploads ADD CONSTRAINT fk_3cbf0b9a2e FOREIGN KEY (batch_id) REFERENCES bulk_import_export_batches(id) ON DELETE CASCADE; +ALTER TABLE ONLY compliance_framework_security_policies + ADD CONSTRAINT fk_3ce58167f1 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY ci_pipelines ADD CONSTRAINT fk_3d34ab2e06 FOREIGN KEY (pipeline_schedule_id) REFERENCES ci_pipeline_schedules(id) ON DELETE SET NULL; @@ -32944,6 +32992,9 @@ ALTER TABLE ONLY projects ALTER TABLE ONLY dast_profile_schedules ADD CONSTRAINT fk_6cca0d8800 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; +ALTER TABLE ONLY compliance_framework_security_policies + ADD CONSTRAINT fk_6d3bd0c9f1 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY vulnerability_merge_request_links ADD CONSTRAINT fk_6d7aa8796e FOREIGN KEY (merge_request_id) REFERENCES merge_requests(id) ON DELETE CASCADE; diff --git a/doc/development/stage_group_observability/img/error_budget_calculation_v17_2.png b/doc/development/stage_group_observability/img/error_budget_calculation_v17_2.png new file mode 100644 index 00000000000..66468e29ef1 Binary files /dev/null and b/doc/development/stage_group_observability/img/error_budget_calculation_v17_2.png differ diff --git a/doc/development/stage_group_observability/index.md b/doc/development/stage_group_observability/index.md index 057a2ecc092..4d04d044394 100644 --- a/doc/development/stage_group_observability/index.md +++ b/doc/development/stage_group_observability/index.md @@ -87,9 +87,16 @@ component can have two indicators: The calculation of the ratio happens as follows: +![error budget calculation](img/error_budget_calculation_v17_2.png) + + ## Check where budget is being spent diff --git a/doc/user/application_security/dast/authentication.md b/doc/user/application_security/dast/authentication.md index 3bc2675d2b1..c9e6e471649 100644 --- a/doc/user/application_security/dast/authentication.md +++ b/doc/user/application_security/dast/authentication.md @@ -70,25 +70,23 @@ Use the following CI/CD variables to configure the authentication actions requir | CI/CD variable | Type | Description | |:------------------------------------|:------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `DAST_AUTH_COOKIES` | string | Set to a comma-separated list of cookie names to specify which cookies are used for authentication. | +| `DAST_AUTH_COOKIE_NAMES` | string | Set to a comma-separated list of cookie names to specify which cookies are used for authentication. | | `DAST_AUTH_REPORT` | boolean | Set to `true` to generate a report detailing steps taken during the authentication process. You must also define `gl-dast-debug-auth-report.html` as a CI job artifact to be able to access the generated report. The report's content aids when debugging authentication failures. | | `DAST_AUTH_TYPE` 1 | string | The authentication type to use. Example: `basic-digest`. | | `DAST_AUTH_URL` | URL | The URL of the page containing the login form on the target website. `DAST_USERNAME` and `DAST_PASSWORD` are submitted with the login form to create an authenticated scan. Example: `https://login.example.com`. | -| `DAST_AUTH_VERIFICATION_LOGIN_FORM` | boolean | Verifies successful authentication by checking for the absence of a login form after the login form has been submitted. | -| `DAST_AUTH_VERIFICATION_SELECTOR` | [selector](#finding-an-elements-selector) | A selector describing an element whose presence is used to determine if authentication has succeeded after the login form is submitted. Example: `css:.user-photo`. | -| `DAST_AUTH_VERIFICATION_URL` | URL | A URL that is compared to the URL in the browser to determine if authentication has succeeded after the login form is submitted. Example: `"https://example.com/loggedin_page"`. | -| `DAST_BROWSER_PATH_TO_LOGIN_FORM` | [selector](#finding-an-elements-selector) | A comma-separated list of selectors representing elements to click on prior to entering the `DAST_USERNAME` and `DAST_PASSWORD` into the login form. Example: `"css:.navigation-menu,css:.login-menu-item"`. | -| `DAST_EXCLUDE_URLS` | URLs | The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. | -| `DAST_FIRST_SUBMIT_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element that is clicked on to submit the username form of a multi-page login process. For example, `css:button[type='user-submit']`. | -| `DAST_PASSWORD` | string | The password to authenticate to in the website. Example: `P@55w0rd!` | -| `DAST_PASSWORD_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element used to enter the password on the login form. Example: `id:password` | -| `DAST_SUBMIT_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element clicked on to submit the login form for a single-page login form, or the password form for a multi-page login form. For example, `css:button[type='submit']`. | -| `DAST_USERNAME` | string | The username to authenticate to in the website. Example: `admin` | -| `DAST_USERNAME_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element used to enter the username on the login form. Example: `name:username` | +| `DAST_AUTH_SUCCESS_IF_NO_LOGIN_FORM` | boolean | Verifies successful authentication by checking for the absence of a login form after the login form has been submitted. This success check is enabled by default. | +| `DAST_AUTH_SUCCESS_IF_ELEMENT_FOUND` | [selector](#finding-an-elements-selector) | A selector describing an element whose presence is used to determine if authentication has succeeded after the login form is submitted. Example: `css:.user-photo`. | +| `DAST_AUTH_SUCCESS_IF_AT_URL` | URL | A URL that is compared to the URL in the browser to determine if authentication has succeeded after the login form is submitted. Example: `"https://example.com/loggedin_page"`. | +| `DAST_AUTH_BEFORE_LOGIN_ACTIONS` | [selector](#finding-an-elements-selector) | A comma-separated list of selectors representing elements to click on prior to entering the `DAST_USERNAME` and `DAST_PASSWORD` into the login form. Example: `"css:.navigation-menu,css:.login-menu-item"`. | +| `DAST_SCOPE_EXCLUDE_URLS` | URLs | The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. | +| `DAST_AUTH_FIRST_SUBMIT_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element that is clicked on to submit the username form of a multi-page login process. For example, `css:button[type='user-submit']`. | +| `DAST_AUTH_PASSWORD` | string | The password to authenticate to in the website. Example: `P@55w0rd!` | +| `DAST_AUTH_PASSWORD_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element used to enter the password on the login form. Example: `id:password` | +| `DAST_AUTH_SUBMIT_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element clicked on to submit the login form for a single-page login form, or the password form for a multi-page login form. For example, `css:button[type='submit']`. | +| `DAST_AUTH_USERNAME` | string | The username to authenticate to in the website. Example: `admin` | +| `DAST_AUTH_USERNAME_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element used to enter the username on the login form. Example: `name:username` | | `DAST_AUTH_DISABLE_CLEAR_FIELDS` | boolean | Disables clearing of username and password fields before attempting manual login. Set to `false` by default. | -| `DAST_AFTER_LOGIN_ACTIONS` | string | Comma separated list of actions to be run after login but before login verification. Currently supports "click" actions. Example: `click(on=id:change_to_bar_graph),click(on=css:input[name=username])` | - -1. Not available to proxy-based scans. +| `DAST_AUTH_AFTER_LOGIN_ACTIONS` | string | Comma separated list of actions to be run after login but before login verification. Currently supports "click" actions. Example: `click(on=id:change_to_bar_graph),click(on=css:input[name=username])` | ### Update the target website diff --git a/doc/user/application_security/dast/authentication_troubleshooting.md b/doc/user/application_security/dast/authentication_troubleshooting.md index 54b9d743abd..ec3f42876d9 100644 --- a/doc/user/application_security/dast/authentication_troubleshooting.md +++ b/doc/user/application_security/dast/authentication_troubleshooting.md @@ -76,7 +76,7 @@ Suggested actions: - Check the target application authentication is deployed and running. - Check the `DAST_AUTH_URL` is correct. - Check the GitLab Runner can access the `DAST_AUTH_URL`. -- Check the `DAST_BROWSER_PATH_TO_LOGIN_FORM` is valid if used. +- Check the `DAST_AUTH_BEFORE_LOGIN_ACTIONS` is valid if used. ### Scan doesn't crawl authenticated pages @@ -92,7 +92,7 @@ Suggested actions: - Generate the [authentication report](#configure-the-authentication-report) and look at the screenshot from the `Login submit` to verify that the login worked as expected. - Verify the logged authentication tokens are those used by your application. -- If using cookies to store authentication tokens, set the names of the authentication token cookies using `DAST_AUTH_COOKIES`. +- If using cookies to store authentication tokens, set the names of the authentication token cookies using `DAST_AUTH_COOKIE_NAMES`. ### Unable to find elements with selector @@ -105,7 +105,7 @@ DAST failed to find the username, password, first submit button, or submit butto Suggested actions: - Generate the [authentication report](#configure-the-authentication-report) to use the screenshot from the `Login page` to verify that the page loaded correctly. -- Load the login page in a browser and verify the [selectors](authentication.md#finding-an-elements-selector) configured in `DAST_USERNAME_FIELD`, `DAST_PASSWORD_FIELD`, `DAST_FIRST_SUBMIT_FIELD`, and `DAST_SUBMIT_FIELD` are correct. +- Load the login page in a browser and verify the [selectors](authentication.md#finding-an-elements-selector) configured in `DAST_AUTH_USERNAME_FIELD`, `DAST_AUTH_PASSWORD_FIELD`, `DAST_AUTH_FIRST_SUBMIT_FIELD`, and `DAST_AUTH_SUBMIT_FIELD` are correct. ### Failed to authenticate user @@ -141,14 +141,14 @@ Suggested actions: - Generate the [authentication report](#configure-the-authentication-report) and verify the `Request` for the `Login submit` is correct. - It's possible that the authentication report `Login submit` request and response are empty. This occurs when there is no request that would result in a full page reload, such as a request made when submitting a HTML form. This occurs when using websockets or AJAX to submit the login form. -- If the page displayed following user authentication genuinely has elements matching the login form selectors, configure `DAST_AUTH_VERIFICATION_URL` - or `DAST_AUTH_VERIFICATION_SELECTOR` to use an alternate method of verifying the login attempt. -- Some applications display a "Loading..." element on a page before hiding the login form. This can confuse the analyzer. Use `DAST_BROWSER_PAGE_LOADING_SELECTOR` or - `DAST_BROWSER_PAGE_READY_SELECTOR` [variable](browser/configuration/variables.md) to instruct the analyzer that the page has finished loading. +- If the page displayed following user authentication genuinely has elements matching the login form selectors, configure `DAST_AUTH_SUCCESS_IF_AT_URL` + or `DAST_AUTH_SUCCESS_IF_ELEMENT_FOUND` to use an alternate method of verifying the login attempt. +- Some applications display a "Loading..." element on a page before hiding the login form. This can confuse the analyzer. Use `DAST_PAGE_IS_LOADING_ELEMENT` or + `DAST_PAGE_IS_READY_ELEMENT` [variable](browser/configuration/variables.md) to instruct the analyzer that the page has finished loading. ### Requirement unsatisfied, selector returned no results -DAST cannot find an element matching the selector provided in `DAST_AUTH_VERIFICATION_SELECTOR` on the page displayed following user login. +DAST cannot find an element matching the selector provided in `DAST_AUTH_SUCCESS_IF_ELEMENT_FOUND` on the page displayed following user login. ```plaintext 2022-12-07T06:39:33.239 INF AUTH requirement is unsatisfied, searching DOM using selector returned no results want="has element css:[name=welcome]" @@ -157,11 +157,11 @@ DAST cannot find an element matching the selector provided in `DAST_AUTH_VERIFIC Suggested actions: - Generate the [authentication report](#configure-the-authentication-report) and look at the screenshot from the `Login submit` to verify that the expected page is displayed. -- Ensure the `DAST_AUTH_VERIFICATION_SELECTOR` [selector](authentication.md#finding-an-elements-selector) is correct. +- Ensure the `DAST_AUTH_SUCCESS_IF_ELEMENT_FOUND` [selector](authentication.md#finding-an-elements-selector) is correct. ### Requirement unsatisfied, browser not at URL -DAST detected that the page displayed following user login has a URL different to what was expected according to `DAST_AUTH_VERIFICATION_URL`. +DAST detected that the page displayed following user login has a URL different to what was expected according to `DAST_AUTH_SUCCESS_IF_AT_URL`. ```plaintext 2022-12-07T11:28:00.241 INF AUTH requirement is unsatisfied, browser is not at URL browser_url="https://example.com/home" want="is at url https://example.com/user/dashboard" @@ -170,7 +170,7 @@ DAST detected that the page displayed following user login has a URL different t Suggested actions: - Generate the [authentication report](#configure-the-authentication-report) and look at the screenshot from the `Login submit` to verify that the expected page is displayed. -- Ensure the `DAST_AUTH_VERIFICATION_URL` is correct. +- Ensure the `DAST_AUTH_SUCCESS_IF_AT_URL` is correct. ### Requirement unsatisfied, HTTP login request status code @@ -199,4 +199,4 @@ Suggestion actions: - Generate the [authentication report](#configure-the-authentication-report) and look at the screenshot from the `Login submit` to verify that the login worked as expected. - Using the browser's developer tools, investigate the cookies and local/session storage objects created while logging in. Ensure there is an authentication token created with sufficiently random value. -- If using cookies to store authentication tokens, set the names of the authentication token cookies using `DAST_AUTH_COOKIES`. +- If using cookies to store authentication tokens, set the names of the authentication token cookies using `DAST_AUTH_COOKIE_NAMES`. diff --git a/doc/user/application_security/dast/browser/configuration/customize_settings.md b/doc/user/application_security/dast/browser/configuration/customize_settings.md index 733c4edf43e..d96f73e7c13 100644 --- a/doc/user/application_security/dast/browser/configuration/customize_settings.md +++ b/doc/user/application_security/dast/browser/configuration/customize_settings.md @@ -100,7 +100,7 @@ This can come at a cost of increased scan time. You can manage the trade-off between coverage and scan time with the following measures: -- Vertically scale the runner and use a higher number of browsers with the [variable](variables.md) `DAST_CRAWL_WORKER_COUNT`. The default is `3`. +- Vertically scale the runner and use a higher number of browsers with the [variable](variables.md) `DAST_CRAWL_WORKER_COUNT`. The default is dynamically set to the number of usable logical CPUs. - Limit the number of actions executed by the browser with the [variable](variables.md) `DAST_CRAWL_MAX_ACTIONS`. The default is `10,000`. - Limit the page depth that the browser-based crawler checks coverage on with the [variable](variables.md) `DAST_CRAWL_MAX_DEPTH`. The crawler uses a breadth-first search strategy, so pages with smaller depth are crawled first. The default is `10`. - Limit the time taken to crawl the target application with the [variable](variables.md) `DAST_CRAWL_TIMEOUT`. The default is `24h`. Scans continue with passive and active checks when the crawler times out. diff --git a/doc/user/application_security/dast/browser/configuration/variables.md b/doc/user/application_security/dast/browser/configuration/variables.md index fc5ddcb4ab3..41253cac9ea 100644 --- a/doc/user/application_security/dast/browser/configuration/variables.md +++ b/doc/user/application_security/dast/browser/configuration/variables.md @@ -26,7 +26,7 @@ For authentication CI/CD variables, see [Authentication](authentication.md). | `DAST_AUTH_SUBMIT_FIELD` | [selector](authentication.md#finding-an-elements-selector) | `css:input[type=submit]` | A selector describing the element clicked on to submit the login form for a single-page login form, or the password form for a multi-page login form. | | `DAST_AUTH_SUCCESS_IF_AT_URL` | URL | `https://www.site.com/welcome` | A URL that is compared to the URL in the browser to determine if authentication has succeeded after the login form is submitted. | | `DAST_AUTH_SUCCESS_IF_ELEMENT_FOUND` | [selector](authentication.md#finding-an-elements-selector) | `css:.user-avatar` | A selector describing an element whose presence is used to determine if authentication has succeeded after the login form is submitted. | -| `DAST_AUTH_SUCCESS_IF_NO_LOGIN_FORM` | boolean | `true` | Verifies successful authentication by checking for the absence of a login form after the login form has been submitted. | +| `DAST_AUTH_SUCCESS_IF_NO_LOGIN_FORM` | boolean | `true` | Verifies successful authentication by checking for the absence of a login form after the login form has been submitted. This success check is enabled by default. | | `DAST_AUTH_TYPE` | string | `basic-digest` | The authentication type to use. | | `DAST_AUTH_URL` | URL | `https://site.com/login` | The URL of the page containing the login form on the target website. `DAST_AUTH_USERNAME` and `DAST_AUTH_PASSWORD` are submitted with the login form to create an authenticated scan. | | `DAST_AUTH_USERNAME_FIELD` | [selector](authentication.md#finding-an-elements-selector) | `name:username` | A selector describing the element used to enter the username on the login form. | diff --git a/doc/user/application_security/dast/browser/troubleshooting.md b/doc/user/application_security/dast/browser/troubleshooting.md index c770288301d..3a9b36a7484 100644 --- a/doc/user/application_security/dast/browser/troubleshooting.md +++ b/doc/user/application_security/dast/browser/troubleshooting.md @@ -153,6 +153,7 @@ The modules that can be configured for logging are as follows: | `STAT` | Used for general statistics while running the scan. | | `VLDFN` | Used for loading and parsing vulnerability definitions. | | `WEBGW` | Used to log messages sent to the target application when running active checks. | +| `SCOPE` | Used to log messages related to [scope management](configuration/customize_settings.md#managing-scope). | ### Example - log crawled paths diff --git a/doc/user/application_security/dast/browser_based_4_to_5_migration_guide.md b/doc/user/application_security/dast/browser_based_4_to_5_migration_guide.md index aba3406ce9c..f64224ab217 100644 --- a/doc/user/application_security/dast/browser_based_4_to_5_migration_guide.md +++ b/doc/user/application_security/dast/browser_based_4_to_5_migration_guide.md @@ -79,6 +79,7 @@ See [configuration](browser/configuration/index.md) for more information on conf | DAST version 4 CI/CD variable | Required action | Notes | |:--------------------------------------------|:-------------------|:----------------------------------------------| | `DAST_ADVERTISE_SCAN` | Rename | To `DAST_REQUEST_ADVERTISE_SCAN` | +| `DAST_AFTER_LOGIN_ACTIONS` | Rename | To `DAST_AUTH_AFTER_LOGIN_ACTIONS` | | `DAST_AUTH_COOKIES` | Rename | To `DAST_AUTH_COOKIE_NAMES` | | `DAST_AUTH_DISABLE_CLEAR_FIELDS` | Rename | To `DAST_AUTH_CLEAR_INPUT_FIELDS` | | `DAST_AUTH_REPORT` | No action required | | diff --git a/doc/user/img/markdown_math_v17_2.png b/doc/user/img/markdown_math_v17_2.png new file mode 100644 index 00000000000..d2faa474dd7 Binary files /dev/null and b/doc/user/img/markdown_math_v17_2.png differ diff --git a/doc/user/markdown.md b/doc/user/markdown.md index b77aca81383..942f9568a0b 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -1039,21 +1039,7 @@ a^2+b^2=c^2 $$ ```` -This math is inline: $`a^2+b^2=c^2`$. - -This math is on a separate line using a ```` ```math ```` block: - -```math -a^2+b^2=c^2 -``` - -This math is on a separate line using inline `$$`: $$a^2+b^2=c^2$$ - -This math is on a separate line using a `$$...$$` block: - -$$ -a^2+b^2=c^2 -$$ +![Example of math in GitLab](img/markdown_math_v17_2.png) ## Tables diff --git a/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_namespace_id.rb b/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_namespace_id.rb new file mode 100644 index 00000000000..1f66e024622 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_namespace_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillComplianceFrameworkSecurityPoliciesNamespaceId < BackfillDesiredShardingKeyJob + operation_name :backfill_compliance_framework_security_policies_namespace_id + feature_category :security_policy_management + end + end +end diff --git a/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_project_id.rb b/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_project_id.rb new file mode 100644 index 00000000000..f72e03c6d3f --- /dev/null +++ b/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_project_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillComplianceFrameworkSecurityPoliciesProjectId < BackfillDesiredShardingKeyJob + operation_name :backfill_compliance_framework_security_policies_project_id + feature_category :security_policy_management + end + end +end diff --git a/lib/gitlab/background_migration/deduplicate_lfs_objects_projects.rb b/lib/gitlab/background_migration/deduplicate_lfs_objects_projects.rb new file mode 100644 index 00000000000..e4191946ffd --- /dev/null +++ b/lib/gitlab/background_migration/deduplicate_lfs_objects_projects.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class DeduplicateLfsObjectsProjects < BatchedMigrationJob + operation_name :deduplicates_lfs_objects_projects + feature_category :source_code_management + + # Temporary class to link AR model to the `lfs_objects_projects` table + class LfsObjectsProject < ::ApplicationRecord + include EachBatch + + self.table_name = 'lfs_objects_projects' + end + + def perform + each_sub_batch do |relation| + data = duplicates_by_project_id_and_lfs_object_id(relation) + + next if data.empty? + + # After plucking the duplicates, build a VALUE list + id_list = Arel::Nodes::ValuesList.new(data).to_sql + + # Use the same GROUP BY query as in the MR to properly narrow down the duplicated records. + # In the previous query we didn't include the repository_type because it is not covered with an index. + subquery = LfsObjectsProject + .where("(project_id, lfs_object_id) IN (#{id_list})") # rubocop:disable GitlabSecurity/SqlInjection -- there is no user input given + .select('project_id, lfs_object_id, repository_type, MAX(id) AS max_id') + .group('project_id, lfs_object_id, repository_type') + .having('COUNT(*) > 1') + + join_query = <<~SQL.squish + INNER JOIN (#{subquery.to_sql}) AS duplicates + ON lfs_objects_projects.project_id = duplicates.project_id + AND lfs_objects_projects.lfs_object_id = duplicates.lfs_object_id + AND lfs_objects_projects.repository_type = duplicates.repository_type + SQL + + duplicated_lfs_objects_projects = LfsObjectsProject.joins(join_query).where.not( + 'lfs_objects_projects.id = duplicates.max_id' + ) + + LfsObjectsProject.where(id: duplicated_lfs_objects_projects.select(:id)).delete_all + end + end + + private + + def duplicates_by_project_id_and_lfs_object_id(relation) + # Select project_id and lfs_object_id pairs which have duplicates. + inner_query = LfsObjectsProject + .select('1') + .from('lfs_objects_projects lop') + .where('lop.project_id = lfs_objects_projects.project_id') + .where('lop.lfs_object_id = lfs_objects_projects.lfs_object_id') + .limit(2) + + count_query = LfsObjectsProject.select('COUNT(*) AS count').from("(#{inner_query.to_sql}) cnt") + + cte = Gitlab::SQL::CTE.new(:distinct_values, relation.select(:project_id, :lfs_object_id).distinct) + + # Limit count to determine if there is a duplicate, we don't need to load all duplicated rows + # (only 2 rows are enough for a project_id, lfs_object_id) pair + cte.apply_to(LfsObjectsProject.where({})) + .where("(#{count_query.to_sql}) = 2") # rubocop:disable GitlabSecurity/SqlInjection -- there is no user input given + .pluck(:project_id, :lfs_object_id) + end + end + end +end diff --git a/spec/features/projects/container_registry_spec.rb b/spec/features/projects/container_registry_spec.rb index 6ce2d042467..cd7191f6f01 100644 --- a/spec/features/projects/container_registry_spec.rb +++ b/spec/features/projects/container_registry_spec.rb @@ -24,185 +24,223 @@ RSpec.describe 'Container Registry', :js, feature_category: :container_registry stub_container_registry_config(enabled: true) stub_container_registry_info stub_container_registry_tags(repository: :any, tags: []) - allow(ContainerRegistry::GitlabApiClient).to receive(:supports_gitlab_api?).and_return(true) end - it 'has a page title set' do - visit_container_registry - - expect(page).to have_title _('Container Registry') - end - - it 'has link to next generation container registry docs' do - allow(ContainerRegistry::GitlabApiClient).to receive(:supports_gitlab_api?).and_return(false) - - visit_container_registry - - expect(page).to have_link('next-generation container registry', href: help_page_href) - end - - it 'does not have link to settings' do - visit_container_registry - - expect(page).not_to have_link _('Configure in settings') - end - - it 'has link to settings when user is maintainer' do - project.add_maintainer(user) - - visit_container_registry - - expect(page).to have_link _('Configure in settings') - end - - context 'when there are no image repositories' do - it 'list page has no container title' do - visit_container_registry - - expect(page).to have_content _('There are no container images stored for this project') - end - - it 'list page has cli commands' do - visit_container_registry - - expect(page).to have_content _('CLI Commands') - end - end - - context 'when there are image repositories' do + context 'with metadatabase enabled' do before do - stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest], with_manifest: true) - project.container_repositories << container_repository + allow(ContainerRegistry::GitlabApiClient).to receive(:supports_gitlab_api?).and_return(true) end - it 'list page has a list of images' do + it 'has a page title set' do visit_container_registry - expect(page).to have_content '1 Image repository' - expect(page).to have_content 'my/image' + expect(page).to have_title _('Container Registry') end - it 'user removes entire container repository' do + it 'has link to next generation container registry docs' do + allow(ContainerRegistry::GitlabApiClient).to receive(:supports_gitlab_api?).and_return(false) + visit_container_registry - expect_any_instance_of(ContainerRepository).to receive(:delete_scheduled!).and_call_original - - find('[title="Remove repository"]').click - expect(find('.modal .modal-title')).to have_content _('Delete image repository?') - find('.modal .modal-body input').set('my/image') - find('.modal .modal-footer .btn-danger').click + expect(page).to have_link('next-generation container registry', href: help_page_href) end - it 'navigates to repo details' do - visit_container_registry_details('my/image') + it 'does not have link to settings' do + visit_container_registry - expect(page).to have_content 'latest' + expect(page).not_to have_link _('Configure in settings') end - describe 'image repo details' do + it 'has link to settings when user is maintainer' do + project.add_maintainer(user) + + visit_container_registry + + expect(page).to have_link _('Configure in settings') + end + + context 'when there are no image repositories' do + it 'list page has no container title' do + visit_container_registry + + expect(page).to have_content _('There are no container images stored for this project') + end + + it 'list page has cli commands' do + visit_container_registry + + expect(page).to have_content _('CLI Commands') + end + end + + context 'when there are image repositories' do before do - stub_container_registry_tags(repository: %r{my/image}, tags: ('1'..'20').to_a, with_manifest: true) - visit_container_registry_details 'my/image' - click_sort_option('Name', true) + stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest], with_manifest: true) + project.container_repositories << container_repository end - it 'shows the details breadcrumb' do - expect(find_by_testid('breadcrumb-links')).to have_link 'my/image' - end + it 'list page has a list of images' do + visit_container_registry - it 'shows the image title' do + expect(page).to have_content '1 Image repository' expect(page).to have_content 'my/image' end - it 'shows the image tags' do - expect(page).to have_content '20 tags' - first_tag = first('[data-testid="name"]') - expect(first_tag).to have_content '1' - end + it 'user removes entire container repository' do + visit_container_registry - it 'user removes a specific tag from container repository' do - service = double('service') - expect(service).to receive(:execute).with(container_repository) { { status: :success } } - expect(Projects::ContainerRepository::DeleteTagsService).to receive(:new).with(container_repository.project, user, tags: ['1']) { service } + expect_any_instance_of(ContainerRepository).to receive(:delete_scheduled!).and_call_original - first('[data-testid="additional-actions"]').click - first('[data-testid="single-delete-button"]').click - expect(find('.modal .modal-title')).to have_content _('Remove tag') - stub_container_registry_tags(repository: %r{my/image}, tags: ('1'..'19').to_a, with_manifest: true) + find('[title="Remove repository"]').click + expect(find('.modal .modal-title')).to have_content _('Delete image repository?') + find('.modal .modal-body input').set('my/image') find('.modal .modal-footer .btn-danger').click - - expect(page).to have_content '19 tags' - expect(page).not_to have_content '20 tags' end - it('pagination navigate to the second page') do - visit_next_page + it 'navigates to repo details' do + visit_container_registry_details('my/image') - expect(page).to have_content '20' - end - end - - describe 'with a tag missing digest' do - before do - stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest stable]) - stub_next_container_registry_tags_call(:digest, nil) - visit_container_registry_details 'my/image' + expect(page).to have_content 'latest' end - it 'renders the tags list correctly' do - expect(page).to have_content('latest') - expect(page).to have_content('stable') - expect(page).to have_content('Digest: Not applicable.') - end - end - - [ContainerRegistry::Path::InvalidRegistryPathError, Faraday::Error].each do |error_class| - context "when there is a #{error_class}" do + describe 'image repo details' do before do - expect(::ContainerRegistry::Client).to receive(:registry_info).and_raise(error_class, nil, nil) + stub_container_registry_tags(repository: %r{my/image}, tags: ('1'..'25').to_a, with_manifest: true) + visit_container_registry_details 'my/image' + click_sort_option('Name', true) end - it_behaves_like 'handling feature network errors with the container registry' + it 'shows the details breadcrumb' do + expect(find_by_testid('breadcrumb-links')).to have_link 'my/image' + end + + it 'shows the image title' do + expect(page).to have_content 'my/image' + end + + it 'shows the image tags' do + expect(page).to have_content '25 tags' + first_tag = first('[data-testid="name"]') + expect(first_tag).to have_content '1' + end + + it 'user removes a specific tag from container repository' do + service = double('service') + expect(service).to receive(:execute).with(container_repository) { { status: :success } } + expect(Projects::ContainerRepository::DeleteTagsService).to receive(:new).with(container_repository.project, user, tags: ['1']) { service } + + first('[data-testid="additional-actions"]').click + first('[data-testid="single-delete-button"]').click + expect(find('.modal .modal-title')).to have_content _('Remove tag') + stub_container_registry_tags(repository: %r{my/image}, tags: ('1'..'19').to_a, with_manifest: true) + find('.modal .modal-footer .btn-danger').click + + expect(page).to have_content '19 tags' + expect(page).not_to have_content '20 tags' + end + + it('pagination navigate to the second page') do + visit_next_page + + expect(page).to have_content '20' + end + end + + describe 'with a tag missing digest' do + before do + stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest stable]) + stub_next_container_registry_tags_call(:digest, nil) + visit_container_registry_details 'my/image' + end + + it 'renders the tags list correctly', :aggregate_failures do + expect(page).to have_content('latest') + expect(page).to have_content('stable') + expect(page).to have_content('Digest: Not applicable.') + end + end + + [ContainerRegistry::Path::InvalidRegistryPathError, Faraday::Error].each do |error_class| + context "when there is a #{error_class}" do + before do + expect(::ContainerRegistry::Client).to receive(:registry_info).and_raise(error_class, nil, nil) + end + + it_behaves_like 'handling feature network errors with the container registry' + end + end + end + + describe 'image repo details when image has no name' do + before do + stub_container_registry_tags(tags: %w[latest], with_manifest: true) + project.container_repositories << nameless_container_repository + visit_container_registry + end + + it 'renders correctly' do + find('a[data-testid="details-link"]').click + + expect(page).to have_content 'latest' + end + end + + context 'when there are more than 20 images' do + before do + project.container_repositories << container_repository + create_list(:container_repository, 22, project: project) + + visit_container_registry + end + + it 'shows pagination' do + expect(page).to have_css '.gl-keyset-pagination' + end + + it 'pagination goes to second page' do + visit_next_page + expect(page).to have_content 'my/image' + end + + it 'pagination is preserved after navigating back from details' do + visit_next_page + click_link 'my/image' + page.go_back + expect(page).to have_content 'my/image' end end end - describe 'image repo details when image has no name' do + describe 'with metadatabase disabled' do before do - stub_container_registry_tags(tags: %w[latest], with_manifest: true) - project.container_repositories << nameless_container_repository - visit_container_registry + allow(ContainerRegistry::GitlabApiClient).to receive(:supports_gitlab_api?).and_return(false) end - it 'renders correctly' do - find('a[data-testid="details-link"]').click + context 'when there are more than 10 images' do + before do + project.container_repositories << container_repository + create_list(:container_repository, 12, project: project) - expect(page).to have_content 'latest' - end - end + visit_container_registry + end - context 'when there are more than 10 images' do - before do - project.container_repositories << container_repository - create_list(:container_repository, 12, project: project) + it 'shows pagination' do + expect(page).to have_css '.gl-keyset-pagination' + end - visit_container_registry - end + it 'pagination goes to second page' do + visit_next_page - it 'shows pagination' do - expect(page).to have_css '.gl-keyset-pagination' - end + expect(page).to have_content 'my/image' + end - it 'pagination goes to second page' do - visit_next_page - expect(page).to have_content 'my/image' - end + it 'pagination is preserved after navigating back from details' do + visit_next_page + click_link 'my/image' + page.go_back - it 'pagination is preserved after navigating back from details' do - visit_next_page - click_link 'my/image' - page.go_back - expect(page).to have_content 'my/image' + expect(page).to have_content 'my/image' + end end end diff --git a/spec/frontend/packages_and_registries/container_registry/explorer/components/details_page/tags_list_spec.js b/spec/frontend/packages_and_registries/container_registry/explorer/components/details_page/tags_list_spec.js index 18962be6f05..60554e44215 100644 --- a/spec/frontend/packages_and_registries/container_registry/explorer/components/details_page/tags_list_spec.js +++ b/spec/frontend/packages_and_registries/container_registry/explorer/components/details_page/tags_list_spec.js @@ -16,6 +16,7 @@ import deleteContainerRepositoryTagsMutation from '~/packages_and_registries/con import { GRAPHQL_PAGE_SIZE, + GRAPHQL_PAGE_SIZE_METADATA_ENABLED, NO_TAGS_TITLE, NO_TAGS_MESSAGE, NO_TAGS_MATCHING_FILTERS_TITLE, @@ -230,6 +231,65 @@ describe('Tags List', () => { ], }); }); + + it('increases page size when paginating next', async () => { + findPersistedPagination().vm.$emit('next'); + + await waitForPromises(); + + expect(resolver).toHaveBeenCalledWith({ + ...queryData, + first: GRAPHQL_PAGE_SIZE_METADATA_ENABLED, + after: tagsPageInfo.endCursor, + }); + }); + + it('increases page size when paginating prev', async () => { + findPersistedPagination().vm.$emit('prev'); + + await waitForPromises(); + + expect(resolver).toHaveBeenCalledWith({ + ...queryData, + first: null, + last: GRAPHQL_PAGE_SIZE_METADATA_ENABLED, + before: tagsPageInfo.startCursor, + }); + }); + + it('with before calls resolver with pagination params', async () => { + findPersistedSearch().vm.$emit('update', { + sort: 'NAME_ASC', + filters: [], + pageInfo: { before: tagsPageInfo.startCursor }, + }); + + await waitForPromises(); + + expect(resolver).toHaveBeenLastCalledWith({ + ...queryData, + first: null, + before: tagsPageInfo.startCursor, + last: GRAPHQL_PAGE_SIZE_METADATA_ENABLED, + }); + }); + + it('with after calls resolver with pagination params', async () => { + findPersistedSearch().vm.$emit('update', { + ...queryData, + sort: 'NAME_ASC', + filters: [], + pageInfo: { after: tagsPageInfo.endCursor }, + }); + + await waitForPromises(); + + expect(resolver).toHaveBeenLastCalledWith({ + ...queryData, + first: GRAPHQL_PAGE_SIZE_METADATA_ENABLED, + after: tagsPageInfo.endCursor, + }); + }); }); }); diff --git a/spec/frontend/packages_and_registries/container_registry/explorer/pages/list_spec.js b/spec/frontend/packages_and_registries/container_registry/explorer/pages/list_spec.js index f6788c9e87c..dc5ff35bc6c 100644 --- a/spec/frontend/packages_and_registries/container_registry/explorer/pages/list_spec.js +++ b/spec/frontend/packages_and_registries/container_registry/explorer/pages/list_spec.js @@ -16,6 +16,7 @@ import { DELETE_IMAGE_SUCCESS_MESSAGE, DELETE_IMAGE_ERROR_MESSAGE, GRAPHQL_PAGE_SIZE, + GRAPHQL_PAGE_SIZE_METADATA_ENABLED, SORT_FIELDS, SETTINGS_TEXT, } from '~/packages_and_registries/container_registry/explorer/constants'; @@ -646,6 +647,63 @@ describe('List Page', () => { }), ); }); + + describe('with metadata database enabled', () => { + it.each` + event | expected + ${'prev'} | ${{ before: pageInfo.startCursor, first: null, last: GRAPHQL_PAGE_SIZE_METADATA_ENABLED }} + ${'next'} | ${{ after: pageInfo.endCursor, first: GRAPHQL_PAGE_SIZE_METADATA_ENABLED }} + `('$event event triggers correct page request', async ({ event, expected }) => { + const resolver = jest.fn().mockResolvedValue(graphQLImageListMock); + const detailsResolver = jest + .fn() + .mockResolvedValue(graphQLProjectImageRepositoriesDetailsMock); + const config = { + isMetadataDatabaseEnabled: true, + isGroupPage: false, + }; + + mountComponent({ resolver, detailsResolver, config }); + fireFirstSortUpdate(); + await waitForApolloRequestRender(); + + findPersistedPagination().vm.$emit(event); + await waitForPromises(); + + expect(resolver).toHaveBeenCalledWith(expect.objectContaining(expected)); + expect(detailsResolver).toHaveBeenCalledWith(expect.objectContaining(expected)); + }); + + it.each` + cursor | expected + ${{ before: pageInfo.startCursor }} | ${{ sort: 'UPDATED_DESC', before: pageInfo.startCursor, first: null, last: GRAPHQL_PAGE_SIZE_METADATA_ENABLED }} + ${{ after: pageInfo.endCursor }} | ${{ sort: 'UPDATED_DESC', after: pageInfo.endCursor, first: GRAPHQL_PAGE_SIZE_METADATA_ENABLED }} + `( + 'calls resolver correctly when persisted search returns $cursor', + async ({ cursor, expected }) => { + const resolver = jest.fn().mockResolvedValue(graphQLImageListMock); + const detailsResolver = jest + .fn() + .mockResolvedValue(graphQLProjectImageRepositoriesDetailsMock); + const config = { + isMetadataDatabaseEnabled: true, + isGroupPage: false, + }; + + mountComponent({ resolver, detailsResolver, config }); + + findPersistedSearch().vm.$emit('update', { + sort: 'UPDATED_DESC', + filters: [], + pageInfo: cursor, + }); + await waitForApolloRequestRender(); + + expect(resolver).toHaveBeenCalledWith(expect.objectContaining(expected)); + expect(detailsResolver).toHaveBeenCalledWith(expect.objectContaining(expected)); + }, + ); + }); }); }); diff --git a/spec/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_namespace_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_namespace_id_spec.rb new file mode 100644 index 00000000000..8cf90e02a52 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_namespace_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillComplianceFrameworkSecurityPoliciesNamespaceId, + feature_category: :security_policy_management, + schema: 20240722095916 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :compliance_framework_security_policies } + let(:backfill_column) { :namespace_id } + let(:backfill_via_table) { :security_orchestration_policy_configurations } + let(:backfill_via_column) { :namespace_id } + let(:backfill_via_foreign_key) { :policy_configuration_id } + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_project_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_project_id_spec.rb new file mode 100644 index 00000000000..e1ef60f0638 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_project_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillComplianceFrameworkSecurityPoliciesProjectId, + feature_category: :security_policy_management, + schema: 20240722095911 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :compliance_framework_security_policies } + let(:backfill_column) { :project_id } + let(:backfill_via_table) { :security_orchestration_policy_configurations } + let(:backfill_via_column) { :project_id } + let(:backfill_via_foreign_key) { :policy_configuration_id } + end +end diff --git a/spec/lib/gitlab/background_migration/deduplicate_lfs_objects_projects_spec.rb b/spec/lib/gitlab/background_migration/deduplicate_lfs_objects_projects_spec.rb new file mode 100644 index 00000000000..51fb55ea520 --- /dev/null +++ b/spec/lib/gitlab/background_migration/deduplicate_lfs_objects_projects_spec.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DeduplicateLfsObjectsProjects, feature_category: :source_code_management do + let(:namespace_table) { table(:namespaces) } + let(:projects_table) { table(:projects) } + let(:lfs_objects_table) { table(:lfs_objects) } + let(:lfs_objects_projects_table) { table(:lfs_objects_projects) } + let(:lfs_object_size) { 20 } + + let(:namespace1) { namespace_table.create!(name: 'namespace1', path: 'namespace1') } + let(:namespace2) { namespace_table.create!(name: 'namespace2', path: 'namespace2') } + let(:namespace3) { namespace_table.create!(name: 'namespace3', path: 'namespace3') } + + let(:project1) { projects_table.create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:project2) { projects_table.create!(namespace_id: namespace2.id, project_namespace_id: namespace2.id) } + let(:project3) { projects_table.create!(namespace_id: namespace3.id, project_namespace_id: namespace3.id) } + + let(:lfs_object1) do + lfs_objects_table.create!( + oid: 'f2b0a1e7550e9b718dafc9b525a04879a766de62e4fbdfc46593d47f7ab74636', + size: lfs_object_size + ) + end + + let(:lfs_object2) do + lfs_objects_table.create!( + oid: '004409da2260f89ceaf6d7cd13cbdfdeeeb43b6c1705299e96efbaa659805785', + size: lfs_object_size + ) + end + + let(:lfs_object3) do + lfs_objects_table.create!( + oid: 'c4c65374aa473c94e81810fbb0adb861292661eeb677a197a19a18e52e8eab9c', + size: lfs_object_size + ) + end + + let(:lfs_object4) do + lfs_objects_table.create!( + oid: '7736b17f9fff13bb54b10738acc03daa6c0867b22d29a5f9430cd97a47ebb6a4', + size: lfs_object_size + ) + end + + let(:lfs_object5) do + lfs_objects_table.create!( + oid: '96f74c6fe7a2979eefb9ec74a5dfc6888fb25543cf99b77586b79afea1da6f97', + size: lfs_object_size + ) + end + + let(:lfs_object6) do + lfs_objects_table.create!( + oid: '47997ea7ecff33be61e3ca1cc287ee72a2125161518f1a169f2893a5a82e9d95', + size: lfs_object_size + ) + end + + let!(:duplicated_lfs_objects_project1) do + lfs_objects_projects_table.create!(project_id: project1.id, lfs_object_id: lfs_object1.id, repository_type: 0) + end + + let!(:lfs_objects_project2) do + lfs_objects_projects_table.create!(project_id: project1.id, lfs_object_id: lfs_object1.id, repository_type: 0) + end + + let!(:duplicated_lfs_objects_project3) do + lfs_objects_projects_table.create!(project_id: project2.id, lfs_object_id: lfs_object4.id, repository_type: 0) + end + + let!(:duplicated_lfs_objects_project4) do + lfs_objects_projects_table.create!(project_id: project2.id, lfs_object_id: lfs_object4.id, repository_type: 0) + end + + let!(:lfs_objects_project5) do + lfs_objects_projects_table.create!(project_id: project2.id, lfs_object_id: lfs_object4.id, repository_type: 0) + end + + let!(:lfs_objects_project6) do + lfs_objects_projects_table.create!(project_id: project3.id, lfs_object_id: lfs_object6.id, repository_type: 1) + end + + let(:migration_attrs) do + { + start_id: projects_table.minimum(:id), + end_id: projects_table.maximum(:id), + batch_table: :lfs_objects_projects, + batch_column: :project_id, + sub_batch_size: 3, + pause_ms: 100, + connection: ApplicationRecord.connection + } + end + + subject(:migration) { described_class.new(**migration_attrs) } + + describe '#perform' do + context 'with duplicates' do + it 'deduplicates lfs_objects_projects by lfs_object_id, repository_type and project_id' do + expect { migration.perform }.to change { lfs_objects_projects_table.count }.from(6).to(3) + + expect(lfs_objects_projects_table.all) + .to contain_exactly(lfs_objects_project2, lfs_objects_project5, lfs_objects_project6) + end + end + + context 'without duplicates' do + it 'does not delete any records' do + [ + duplicated_lfs_objects_project1, + duplicated_lfs_objects_project3, + duplicated_lfs_objects_project4 + ].each(&:destroy) + + expect { migration.perform }.not_to change { lfs_objects_table.count } + end + end + end +end diff --git a/spec/lib/gitlab/database/dictionary_spec.rb b/spec/lib/gitlab/database/dictionary_spec.rb index 2e22b9a3510..289ad59800b 100644 --- a/spec/lib/gitlab/database/dictionary_spec.rb +++ b/spec/lib/gitlab/database/dictionary_spec.rb @@ -75,7 +75,7 @@ RSpec.describe Gitlab::Database::Dictionary, feature_category: :database do it 'returns an array of entries having desired sharding key migration job' do entries = dictionary.find_all_having_desired_sharding_key_migration_job expect(entries).to all(be_instance_of(Gitlab::Database::Dictionary::Entry)) - expect(entries).to all(have_attributes(desired_sharding_key_migration_job_name: String)) + expect(entries.map(&:desired_sharding_key_migration_job_name)).to all(be_present) end end diff --git a/spec/migrations/20240722095915_queue_backfill_compliance_framework_security_policies_project_id_spec.rb b/spec/migrations/20240722095915_queue_backfill_compliance_framework_security_policies_project_id_spec.rb new file mode 100644 index 00000000000..f9cd4fad99d --- /dev/null +++ b/spec/migrations/20240722095915_queue_backfill_compliance_framework_security_policies_project_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillComplianceFrameworkSecurityPoliciesProjectId, feature_category: :security_policy_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :compliance_framework_security_policies, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :project_id, + :security_orchestration_policy_configurations, + :project_id, + :policy_configuration_id + ] + ) + } + end + end +end diff --git a/spec/migrations/20240722095920_queue_backfill_compliance_framework_security_policies_namespace_id_spec.rb b/spec/migrations/20240722095920_queue_backfill_compliance_framework_security_policies_namespace_id_spec.rb new file mode 100644 index 00000000000..d640116f42f --- /dev/null +++ b/spec/migrations/20240722095920_queue_backfill_compliance_framework_security_policies_namespace_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillComplianceFrameworkSecurityPoliciesNamespaceId, feature_category: :security_policy_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :compliance_framework_security_policies, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :namespace_id, + :security_orchestration_policy_configurations, + :namespace_id, + :policy_configuration_id + ] + ) + } + end + end +end diff --git a/spec/migrations/20240808125149_queue_deduplicate_lfs_objects_projects_spec.rb b/spec/migrations/20240808125149_queue_deduplicate_lfs_objects_projects_spec.rb new file mode 100644 index 00000000000..930361c7774 --- /dev/null +++ b/spec/migrations/20240808125149_queue_deduplicate_lfs_objects_projects_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueDeduplicateLfsObjectsProjects, feature_category: :source_code_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :lfs_objects_projects, + column_name: :id, + batch_class_name: described_class::BATCH_CLASS_NAME, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end diff --git a/workhorse/_support/lint_last_known_acceptable_go1.21.txt b/workhorse/_support/lint_last_known_acceptable_go1.21.txt index 39d7c7e7fea..0911364d46f 100644 --- a/workhorse/_support/lint_last_known_acceptable_go1.21.txt +++ b/workhorse/_support/lint_last_known_acceptable_go1.21.txt @@ -3,8 +3,6 @@ cmd/gitlab-resize-image/png/reader.go:1:1: package-comments: should have a packa cmd/gitlab-resize-image/png/reader.go:26:1: exported: exported function NewReader should have comment or be unexported (revive) cmd/gitlab-resize-image/png/reader.go:78:17: var-declaration: should omit type []byte from declaration of var magicBytes; it will be inferred from the right-hand side (revive) cmd/gitlab-workhorse/config_test.go:191: cmd/gitlab-workhorse/config_test.go:191: Line contains TODO/BUG/FIXME/NOTE/OPTIMIZE/HACK: "TODO this is meant to be 50*time.Second ..." (godox) -cmd/gitlab-workhorse/jobs_test.go:37:29: response body must be closed (bodyclose) -cmd/gitlab-workhorse/jobs_test.go:42:29: response body must be closed (bodyclose) cmd/gitlab-workhorse/listener.go:26:16: G402: TLS MinVersion too low. (gosec) cmd/gitlab-workhorse/main.go:9:2: G108: Profiling endpoint is automatically exposed on /debug/pprof (gosec) cmd/gitlab-workhorse/main.go:73: Function 'buildConfig' has too many statements (63 > 40) (funlen) @@ -22,30 +20,16 @@ cmd/gitlab-workhorse/main.go:233:5: shadow: declaration of "err" shadows declara cmd/gitlab-workhorse/main.go:241:26: Error return value of `accessCloser.Close` is not checked (errcheck) cmd/gitlab-workhorse/main.go:265:10: G112: Potential Slowloris Attack because ReadHeaderTimeout is not configured in the http.Server (gosec) cmd/gitlab-workhorse/main_test.go:60:2: exitAfterDefer: os.Exit will exit, and `defer gitaly.CloseConnections()` will not run (gocritic) -cmd/gitlab-workhorse/main_test.go:107:24: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:143:24: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:156:66: wrapperFunc: use http.NotFoundHandler method in `http.HandlerFunc(http.NotFound)` (gocritic) -cmd/gitlab-workhorse/main_test.go:163:23: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:186:24: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:212:25: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:243:23: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:246:25: unnecessary conversion (unconvert) -cmd/gitlab-workhorse/main_test.go:416:38: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:431:38: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:457:3: bool-compare: use assert.True (testifylint) -cmd/gitlab-workhorse/main_test.go:485:40: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:509:23: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:519:3: ifElseChain: rewrite if-else to switch statement (gocritic) -cmd/gitlab-workhorse/main_test.go:575:21: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:578:3: var-naming: var requestIds should be requestIDs (revive) -cmd/gitlab-workhorse/main_test.go:579:3: len: use require.Len (testifylint) -cmd/gitlab-workhorse/main_test.go:651:4: var-naming: var propagatedRequestId should be propagatedRequestID (revive) -cmd/gitlab-workhorse/main_test.go:658:22: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:659:4: var-naming: var requestIds should be requestIDs (revive) -cmd/gitlab-workhorse/main_test.go:662:4: len: use require.Len (testifylint) -cmd/gitlab-workhorse/main_test.go:871:25: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:896:25: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:937:4: var-naming: var originResourceUrl should be originResourceURL (revive) +cmd/gitlab-workhorse/main_test.go:158:66: wrapperFunc: use http.NotFoundHandler method in `http.HandlerFunc(http.NotFound)` (gocritic) +cmd/gitlab-workhorse/main_test.go:252:25: unnecessary conversion (unconvert) +cmd/gitlab-workhorse/main_test.go:467:3: bool-compare: use assert.True (testifylint) +cmd/gitlab-workhorse/main_test.go:532:3: ifElseChain: rewrite if-else to switch statement (gocritic) +cmd/gitlab-workhorse/main_test.go:592:3: var-naming: var requestIds should be requestIDs (revive) +cmd/gitlab-workhorse/main_test.go:593:3: len: use require.Len (testifylint) +cmd/gitlab-workhorse/main_test.go:665:4: var-naming: var propagatedRequestId should be propagatedRequestID (revive) +cmd/gitlab-workhorse/main_test.go:674:4: var-naming: var requestIds should be requestIDs (revive) +cmd/gitlab-workhorse/main_test.go:677:4: len: use require.Len (testifylint) +cmd/gitlab-workhorse/main_test.go:954:4: var-naming: var originResourceUrl should be originResourceURL (revive) cmd/gitlab-workhorse/proxy_test.go:55:9: shadow: declaration of "err" shadows declaration at line 36 (govet) cmd/gitlab-workhorse/proxy_test.go:77:6: var-naming: var tsUrl should be tsURL (revive) cmd/gitlab-workhorse/proxy_test.go:87:6: shadow: declaration of "err" shadows declaration at line 78 (govet) @@ -107,8 +91,7 @@ internal/dependencyproxy/dependencyproxy.go:78: Function 'Inject' is too long (7 internal/dependencyproxy/dependencyproxy.go:116:32: `cancelled` is a misspelling of `canceled` (misspell) internal/dependencyproxy/dependencyproxy_test.go:376:4: go-require: do not use assert.FailNow in http handlers (testifylint) internal/dependencyproxy/dependencyproxy_test.go:403:33: `artifically` is a misspelling of `artificially` (misspell) -internal/dependencyproxy/dependencyproxy_test.go:415:27: response body must be closed (bodyclose) -internal/dependencyproxy/dependencyproxy_test.go:426: internal/dependencyproxy/dependencyproxy_test.go:426: Line contains TODO/BUG/FIXME/NOTE/OPTIMIZE/HACK: "note that the timeout duration here is s..." (godox) +internal/dependencyproxy/dependencyproxy_test.go:428: internal/dependencyproxy/dependencyproxy_test.go:428: Line contains TODO/BUG/FIXME/NOTE/OPTIMIZE/HACK: "note that the timeout duration here is s..." (godox) internal/git/archive.go:35:2: var-naming: struct field CommitId should be CommitID (revive) internal/git/archive.go:43:2: exported: exported var SendArchive should have comment or be unexported (revive) internal/git/archive.go:53: Function 'Inject' has too many statements (47 > 40) (funlen) @@ -177,15 +160,6 @@ internal/imageresizer/image_resizer.go:350:17: Error return value of `res.Body.C internal/imageresizer/image_resizer.go:356:15: G304: Potential file inclusion via variable (gosec) internal/imageresizer/image_resizer.go:363:13: Error return value of `file.Close` is not checked (errcheck) internal/imageresizer/image_resizer_caching.go:6:1: ST1000: at least one file in a package should have a package comment (stylecheck) -internal/imageresizer/image_resizer_test.go:76:30: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:93:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:108:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:120:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:130:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:140:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:148:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:193:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:216:28: response body must be closed (bodyclose) internal/lsif_transformer/parser/docs.go:36:2: var-naming: struct field RangeIds should be RangeIDs (revive) internal/lsif_transformer/parser/parser.go:90:1: exported: exported method Parser.Close should have comment or be unexported (revive) internal/lsif_transformer/parser/ranges.go:35:2: var-naming: struct field RangeIds should be RangeIDs (revive) diff --git a/workhorse/_support/lint_last_known_acceptable_go1.22.txt b/workhorse/_support/lint_last_known_acceptable_go1.22.txt index 36d0e30dcd7..98734858876 100644 --- a/workhorse/_support/lint_last_known_acceptable_go1.22.txt +++ b/workhorse/_support/lint_last_known_acceptable_go1.22.txt @@ -3,8 +3,6 @@ cmd/gitlab-resize-image/png/reader.go:1:1: package-comments: should have a packa cmd/gitlab-resize-image/png/reader.go:26:1: exported: exported function NewReader should have comment or be unexported (revive) cmd/gitlab-resize-image/png/reader.go:78:17: var-declaration: should omit type []byte from declaration of var magicBytes; it will be inferred from the right-hand side (revive) cmd/gitlab-workhorse/config_test.go:191: cmd/gitlab-workhorse/config_test.go:191: Line contains TODO/BUG/FIXME/NOTE/OPTIMIZE/HACK: "TODO this is meant to be 50*time.Second ..." (godox) -cmd/gitlab-workhorse/jobs_test.go:37:29: response body must be closed (bodyclose) -cmd/gitlab-workhorse/jobs_test.go:42:29: response body must be closed (bodyclose) cmd/gitlab-workhorse/listener.go:26:16: G402: TLS MinVersion too low. (gosec) cmd/gitlab-workhorse/main.go:9:2: G108: Profiling endpoint is automatically exposed on /debug/pprof (gosec) cmd/gitlab-workhorse/main.go:73: Function 'buildConfig' has too many statements (63 > 40) (funlen) @@ -22,30 +20,16 @@ cmd/gitlab-workhorse/main.go:233:5: shadow: declaration of "err" shadows declara cmd/gitlab-workhorse/main.go:241:26: Error return value of `accessCloser.Close` is not checked (errcheck) cmd/gitlab-workhorse/main.go:265:10: G112: Potential Slowloris Attack because ReadHeaderTimeout is not configured in the http.Server (gosec) cmd/gitlab-workhorse/main_test.go:60:2: exitAfterDefer: os.Exit will exit, and `defer gitaly.CloseConnections()` will not run (gocritic) -cmd/gitlab-workhorse/main_test.go:107:24: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:143:24: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:156:66: wrapperFunc: use http.NotFoundHandler method in `http.HandlerFunc(http.NotFound)` (gocritic) -cmd/gitlab-workhorse/main_test.go:163:23: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:186:24: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:212:25: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:243:23: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:246:25: unnecessary conversion (unconvert) -cmd/gitlab-workhorse/main_test.go:416:38: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:431:38: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:457:3: bool-compare: use assert.True (testifylint) -cmd/gitlab-workhorse/main_test.go:485:40: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:509:23: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:519:3: ifElseChain: rewrite if-else to switch statement (gocritic) -cmd/gitlab-workhorse/main_test.go:575:21: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:578:3: var-naming: var requestIds should be requestIDs (revive) -cmd/gitlab-workhorse/main_test.go:579:3: len: use require.Len (testifylint) -cmd/gitlab-workhorse/main_test.go:651:4: var-naming: var propagatedRequestId should be propagatedRequestID (revive) -cmd/gitlab-workhorse/main_test.go:658:22: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:659:4: var-naming: var requestIds should be requestIDs (revive) -cmd/gitlab-workhorse/main_test.go:662:4: len: use require.Len (testifylint) -cmd/gitlab-workhorse/main_test.go:871:25: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:896:25: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:937:4: var-naming: var originResourceUrl should be originResourceURL (revive) +cmd/gitlab-workhorse/main_test.go:158:66: wrapperFunc: use http.NotFoundHandler method in `http.HandlerFunc(http.NotFound)` (gocritic) +cmd/gitlab-workhorse/main_test.go:252:25: unnecessary conversion (unconvert) +cmd/gitlab-workhorse/main_test.go:467:3: bool-compare: use assert.True (testifylint) +cmd/gitlab-workhorse/main_test.go:532:3: ifElseChain: rewrite if-else to switch statement (gocritic) +cmd/gitlab-workhorse/main_test.go:592:3: var-naming: var requestIds should be requestIDs (revive) +cmd/gitlab-workhorse/main_test.go:593:3: len: use require.Len (testifylint) +cmd/gitlab-workhorse/main_test.go:665:4: var-naming: var propagatedRequestId should be propagatedRequestID (revive) +cmd/gitlab-workhorse/main_test.go:674:4: var-naming: var requestIds should be requestIDs (revive) +cmd/gitlab-workhorse/main_test.go:677:4: len: use require.Len (testifylint) +cmd/gitlab-workhorse/main_test.go:954:4: var-naming: var originResourceUrl should be originResourceURL (revive) cmd/gitlab-workhorse/proxy_test.go:55:9: shadow: declaration of "err" shadows declaration at line 36 (govet) cmd/gitlab-workhorse/proxy_test.go:77:6: var-naming: var tsUrl should be tsURL (revive) cmd/gitlab-workhorse/proxy_test.go:87:6: shadow: declaration of "err" shadows declaration at line 78 (govet) @@ -107,8 +91,7 @@ internal/dependencyproxy/dependencyproxy.go:78: Function 'Inject' is too long (7 internal/dependencyproxy/dependencyproxy.go:116:32: `cancelled` is a misspelling of `canceled` (misspell) internal/dependencyproxy/dependencyproxy_test.go:376:4: go-require: do not use assert.FailNow in http handlers (testifylint) internal/dependencyproxy/dependencyproxy_test.go:403:33: `artifically` is a misspelling of `artificially` (misspell) -internal/dependencyproxy/dependencyproxy_test.go:415:27: response body must be closed (bodyclose) -internal/dependencyproxy/dependencyproxy_test.go:426: internal/dependencyproxy/dependencyproxy_test.go:426: Line contains TODO/BUG/FIXME/NOTE/OPTIMIZE/HACK: "note that the timeout duration here is s..." (godox) +internal/dependencyproxy/dependencyproxy_test.go:428: internal/dependencyproxy/dependencyproxy_test.go:428: Line contains TODO/BUG/FIXME/NOTE/OPTIMIZE/HACK: "note that the timeout duration here is s..." (godox) internal/git/archive.go:35:2: var-naming: struct field CommitId should be CommitID (revive) internal/git/archive.go:43:2: exported: exported var SendArchive should have comment or be unexported (revive) internal/git/archive.go:53: Function 'Inject' has too many statements (47 > 40) (funlen) @@ -177,15 +160,6 @@ internal/imageresizer/image_resizer.go:350:17: Error return value of `res.Body.C internal/imageresizer/image_resizer.go:356:15: G304: Potential file inclusion via variable (gosec) internal/imageresizer/image_resizer.go:363:13: Error return value of `file.Close` is not checked (errcheck) internal/imageresizer/image_resizer_caching.go:6:1: ST1000: at least one file in a package should have a package comment (stylecheck) -internal/imageresizer/image_resizer_test.go:76:30: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:93:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:108:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:120:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:130:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:140:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:148:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:193:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:216:28: response body must be closed (bodyclose) internal/lsif_transformer/parser/docs.go:36:2: var-naming: struct field RangeIds should be RangeIDs (revive) internal/lsif_transformer/parser/parser.go:90:1: exported: exported method Parser.Close should have comment or be unexported (revive) internal/lsif_transformer/parser/ranges.go:35:2: var-naming: struct field RangeIds should be RangeIDs (revive) diff --git a/workhorse/cmd/gitlab-workhorse/jobs_test.go b/workhorse/cmd/gitlab-workhorse/jobs_test.go index 7846788bae4..2a78e03dd83 100644 --- a/workhorse/cmd/gitlab-workhorse/jobs_test.go +++ b/workhorse/cmd/gitlab-workhorse/jobs_test.go @@ -35,11 +35,13 @@ func testJobsLongPolling(t *testing.T, pollingDuration time.Duration, requestJob func testJobsLongPollingEndpointDisabled(t *testing.T, requestJob requestJobFunction) { resp := testJobsLongPolling(t, 0, requestJob) + defer resp.Body.Close() require.NotEqual(t, "yes", resp.Header.Get("Gitlab-Ci-Builds-Polling")) } func testJobsLongPollingEndpoint(t *testing.T, requestJob requestJobFunction) { resp := testJobsLongPolling(t, time.Minute, requestJob) + defer resp.Body.Close() require.Equal(t, "yes", resp.Header.Get("Gitlab-Ci-Builds-Polling")) } diff --git a/workhorse/cmd/gitlab-workhorse/main_test.go b/workhorse/cmd/gitlab-workhorse/main_test.go index 763fd668319..08ccb60859c 100644 --- a/workhorse/cmd/gitlab-workhorse/main_test.go +++ b/workhorse/cmd/gitlab-workhorse/main_test.go @@ -105,6 +105,7 @@ func TestRegularProjectsAPI(t *testing.T) { "/api/v3/projects/foo%2Fbar%2Fbaz%2Fqux/repository/not/special", } { resp, body := httpGet(t, ws.URL+resource, nil) + defer resp.Body.Close() require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resource) require.Equal(t, apiResponse, body, "GET %q: response body", resource) @@ -141,6 +142,7 @@ func TestAllowedStaticFile(t *testing.T) { "/static file.txt", } { resp, body := httpGet(t, ws.URL+resource, nil) + defer resp.Body.Close() require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resource) require.Equal(t, content, body, "GET %q: response body", resource) @@ -161,6 +163,7 @@ func TestStaticFileRelativeURL(t *testing.T) { resource := "/my-relative-url/static.txt" resp, body := httpGet(t, ws.URL+resource, nil) + defer resp.Body.Close() require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resource) require.Equal(t, content, body, "GET %q: response body", resource) @@ -184,6 +187,7 @@ func TestAllowedPublicUploadsFile(t *testing.T) { "/uploads/static file.txt", } { resp, body := httpGet(t, ws.URL+resource, nil) + defer resp.Body.Close() require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resource) require.Equal(t, content, body, "GET %q: response body", resource) @@ -210,6 +214,7 @@ func TestDeniedPublicUploadsFile(t *testing.T) { } { t.Run(resource, func(t *testing.T) { resp, body := httpGet(t, ws.URL+resource, nil) + defer resp.Body.Close() require.Equal(t, 404, resp.StatusCode, "GET %q: status code", resource) require.Equal(t, "", body, "GET %q: response body", resource) @@ -241,6 +246,7 @@ This is a static error page for code 499 resourcePath := "/error-499" resp, body := httpGet(t, ws.URL+resourcePath, nil) + defer resp.Body.Close() require.Equal(t, 499, resp.StatusCode, "GET %q: status code", resourcePath) require.Equal(t, string(errorPageBody), body, "GET %q: response body", resourcePath) @@ -415,6 +421,7 @@ func TestArtifactsGetSingleFile(t *testing.T) { resp, body, err := doSendDataRequest(t, resourcePath, "artifacts-entry", jsonParams) require.NoError(t, err) + defer resp.Body.Close() require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resourcePath) require.Equal(t, fileContents, string(body), "GET %q: response body", resourcePath) @@ -429,7 +436,10 @@ func TestImageResizing(t *testing.T) { resourcePath := "/uploads/-/system/user/avatar/123/avatar.png?width=40" resp, body, err := doSendDataRequest(t, resourcePath, "send-scaled-img", jsonParams) + require.NoError(t, err, "send resize request") + defer resp.Body.Close() + require.Equal(t, 200, resp.StatusCode, "GET %q: body: %s", resourcePath, body) img, err := png.Decode(bytes.NewReader(body)) @@ -485,6 +495,8 @@ func TestSendURLForArtifacts(t *testing.T) { resp, body, err := doSendDataRequest(t, resourcePath, "send-url", jsonParams) require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode, "GET %q: status code", resourcePath) require.Equal(t, int64(tc.contentLength), resp.ContentLength, "GET %q: Content-Length", resourcePath) require.Equal(t, tc.transferEncoding, resp.TransferEncoding, "GET %q: Transfer-Encoding", resourcePath) @@ -507,6 +519,7 @@ func TestApiContentTypeBlock(t *testing.T) { resourcePath := "/something" resp, body := httpGet(t, ws.URL+resourcePath, nil) + defer resp.Body.Close() require.Equal(t, 500, resp.StatusCode, "GET %q: status code", resourcePath) require.NotContains(t, wrongResponse, body, "GET %q: response body", resourcePath) @@ -573,6 +586,7 @@ func TestCorrelationIdHeader(t *testing.T) { "/api/v3/projects/123/repository/not/special", } { resp, _ := httpGet(t, ws.URL+resource, nil) + defer resp.Body.Close() require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resource) requestIds := resp.Header["X-Request-Id"] @@ -656,6 +670,7 @@ func TestPropagateCorrelationIdHeader(t *testing.T) { } resp, _ := httpGet(t, ws.URL+resource, headers) + defer resp.Body.Close() requestIds := resp.Header["X-Request-Id"] require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resource) @@ -869,6 +884,7 @@ This is a static error page for code 503 } { t.Run(resource, func(t *testing.T) { resp, body := httpGet(t, ws.URL+resource, nil) + defer resp.Body.Close() require.Equal(t, 503, resp.StatusCode, "status code") require.Equal(t, apiResponse, body, "response body") @@ -894,6 +910,7 @@ func TestHealthChecksUnreachable(t *testing.T) { for _, tc := range testCases { t.Run(tc.path, func(t *testing.T) { resp, body := httpGet(t, ws.URL+tc.path, nil) + defer resp.Body.Close() require.Equal(t, 502, resp.StatusCode, "status code") require.Equal(t, tc.responseType, resp.Header.Get("Content-Type"), "content-type") diff --git a/workhorse/internal/dependencyproxy/dependencyproxy_test.go b/workhorse/internal/dependencyproxy/dependencyproxy_test.go index ccd5bd7aa5e..2e85372374c 100644 --- a/workhorse/internal/dependencyproxy/dependencyproxy_test.go +++ b/workhorse/internal/dependencyproxy/dependencyproxy_test.go @@ -415,6 +415,8 @@ func TestLongUploadRequest(t *testing.T) { res, err := rt.RoundTrip(r) assert.NoError(t, err, "RoundTripper should not receive an error") + defer res.Body.Close() + assert.Equal(t, http.StatusOK, res.StatusCode, "RoundTripper should receive a 200 status code") w.WriteHeader(res.StatusCode) } diff --git a/workhorse/internal/imageresizer/image_resizer_test.go b/workhorse/internal/imageresizer/image_resizer_test.go index f16df77f002..7edcac34e06 100644 --- a/workhorse/internal/imageresizer/image_resizer_test.go +++ b/workhorse/internal/imageresizer/image_resizer_test.go @@ -74,6 +74,7 @@ func TestRequestScaledImageFromPath(t *testing.T) { params := resizeParams{Location: tc.imagePath, ContentType: tc.contentType, Width: 64} resp := requestScaledImage(t, nil, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) bounds := imageFromResponse(t, resp).Bounds() @@ -91,6 +92,7 @@ func TestRequestScaledImageWithConditionalGetAndImageNotChanged(t *testing.T) { header.Set("If-Modified-Since", httpTimeStr(clientTime)) resp := requestScaledImage(t, header, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusNotModified, resp.StatusCode) require.Equal(t, httpTimeStr(testImageLastModified(t)), resp.Header.Get("Last-Modified")) require.Empty(t, resp.Header.Get("Content-Type")) @@ -106,6 +108,7 @@ func TestRequestScaledImageWithConditionalGetAndImageChanged(t *testing.T) { header.Set("If-Modified-Since", httpTimeStr(clientTime)) resp := requestScaledImage(t, header, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) require.Equal(t, httpTimeStr(testImageLastModified(t)), resp.Header.Get("Last-Modified")) } @@ -118,6 +121,7 @@ func TestRequestScaledImageWithConditionalGetAndInvalidClientTime(t *testing.T) header.Set("If-Modified-Since", "0") resp := requestScaledImage(t, header, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) require.Equal(t, httpTimeStr(testImageLastModified(t)), resp.Header.Get("Last-Modified")) } @@ -128,6 +132,7 @@ func TestServeOriginalImageWhenSourceImageTooLarge(t *testing.T) { params := resizeParams{Location: imagePath, ContentType: "image/png", Width: 64} resp := requestScaledImage(t, nil, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) img := imageFromResponse(t, resp) @@ -138,6 +143,7 @@ func TestFailFastOnOpenStreamFailure(t *testing.T) { cfg := config.DefaultImageResizerConfig params := resizeParams{Location: "does_not_exist.png", ContentType: "image/png", Width: 64} resp := requestScaledImage(t, nil, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusInternalServerError, resp.StatusCode) } @@ -146,6 +152,7 @@ func TestIgnoreContentTypeMismatchIfImageFormatIsAllowed(t *testing.T) { cfg := config.DefaultImageResizerConfig params := resizeParams{Location: imagePath, ContentType: "image/jpeg", Width: 64} resp := requestScaledImage(t, nil, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) bounds := imageFromResponse(t, resp).Bounds() @@ -191,6 +198,7 @@ func TestServeOriginalImageWhenSourceImageFormatIsNotAllowed(t *testing.T) { params := resizeParams{Location: svgImagePath, ContentType: "image/png", Width: 64} resp := requestScaledImage(t, nil, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) responseData, err := io.ReadAll(resp.Body) @@ -214,6 +222,7 @@ func TestServeOriginalImageWhenSourceImageIsTooSmall(t *testing.T) { params := resizeParams{Location: img.Name(), ContentType: "image/png", Width: 64} resp := requestScaledImage(t, nil, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) responseData, err := io.ReadAll(resp.Body)