From 052e379b75a7f55b6105d063d9520dc790428f7e Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 9 Aug 2024 03:09:38 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .../components/details_page/tags_list.vue | 14 +- .../explorer/constants/list.js | 1 + .../explorer/pages/list.vue | 14 +- .../container_registry/explorer/utils.js | 14 +- app/views/projects/empty.html.haml | 8 +- ...amework_security_policies_namespace_id.yml | 9 + ...framework_security_policies_project_id.yml | 9 + .../deduplicate_lfs_objects_projects.yml | 11 + ...compliance_framework_security_policies.yml | 3 + ..._compliance_framework_security_policies.rb | 9 + ..._compliance_framework_security_policies.rb | 9 + ...dd_index_to_member_roles_on_permissions.rb | 16 + ...ility_count_to_project_statistics_table.rb | 9 + ...amework_security_policies_on_project_id.rb | 16 + ...amework_security_policies_project_id_fk.rb | 17 + ...rk_security_policies_project_id_trigger.rb | 25 ++ ..._framework_security_policies_project_id.rb | 40 +++ ...ework_security_policies_on_namespace_id.rb | 16 + ...ework_security_policies_namespace_id_fk.rb | 17 + ..._security_policies_namespace_id_trigger.rb | 25 ++ ...ramework_security_policies_namespace_id.rb | 40 +++ ..._queue_deduplicate_lfs_objects_projects.rb | 30 ++ db/schema_migrations/20240722095911 | 1 + db/schema_migrations/20240722095912 | 1 + db/schema_migrations/20240722095913 | 1 + db/schema_migrations/20240722095914 | 1 + db/schema_migrations/20240722095915 | 1 + db/schema_migrations/20240722095916 | 1 + db/schema_migrations/20240722095917 | 1 + db/schema_migrations/20240722095918 | 1 + db/schema_migrations/20240722095919 | 1 + db/schema_migrations/20240722095920 | 1 + db/schema_migrations/20240806161731 | 1 + db/schema_migrations/20240807103912 | 1 + db/schema_migrations/20240808125149 | 1 + db/structure.sql | 55 +++- .../img/error_budget_calculation_v17_2.png | Bin 0 -> 10789 bytes .../stage_group_observability/index.md | 7 + .../dast/authentication.md | 28 +- .../dast/authentication_troubleshooting.md | 24 +- .../configuration/customize_settings.md | 2 +- .../dast/browser/configuration/variables.md | 2 +- .../dast/browser/troubleshooting.md | 1 + .../browser_based_4_to_5_migration_guide.md | 1 + doc/user/img/markdown_math_v17_2.png | Bin 0 -> 16634 bytes doc/user/markdown.md | 16 +- ...ramework_security_policies_namespace_id.rb | 10 + ..._framework_security_policies_project_id.rb | 10 + .../deduplicate_lfs_objects_projects.rb | 71 ++++ .../projects/container_registry_spec.rb | 302 ++++++++++-------- .../components/details_page/tags_list_spec.js | 60 ++++ .../explorer/pages/list_spec.js | 58 ++++ ...ork_security_policies_namespace_id_spec.rb | 15 + ...ework_security_policies_project_id_spec.rb | 15 + .../deduplicate_lfs_objects_projects_spec.rb | 122 +++++++ spec/lib/gitlab/database/dictionary_spec.rb | 2 +- ...ework_security_policies_project_id_spec.rb | 33 ++ ...ork_security_policies_namespace_id_spec.rb | 33 ++ ...e_deduplicate_lfs_objects_projects_spec.rb | 26 ++ .../lint_last_known_acceptable_go1.21.txt | 48 +-- .../lint_last_known_acceptable_go1.22.txt | 48 +-- workhorse/cmd/gitlab-workhorse/jobs_test.go | 2 + workhorse/cmd/gitlab-workhorse/main_test.go | 17 + .../dependencyproxy/dependencyproxy_test.go | 2 + .../imageresizer/image_resizer_test.go | 9 + 65 files changed, 1112 insertions(+), 272 deletions(-) create mode 100644 db/docs/batched_background_migrations/backfill_compliance_framework_security_policies_namespace_id.yml create mode 100644 db/docs/batched_background_migrations/backfill_compliance_framework_security_policies_project_id.yml create mode 100644 db/docs/batched_background_migrations/deduplicate_lfs_objects_projects.yml create mode 100644 db/migrate/20240722095911_add_project_id_to_compliance_framework_security_policies.rb create mode 100644 db/migrate/20240722095916_add_namespace_id_to_compliance_framework_security_policies.rb create mode 100644 db/migrate/20240806161731_add_index_to_member_roles_on_permissions.rb create mode 100644 db/migrate/20240807103912_add_vulnerability_count_to_project_statistics_table.rb create mode 100644 db/post_migrate/20240722095912_index_compliance_framework_security_policies_on_project_id.rb create mode 100644 db/post_migrate/20240722095913_add_compliance_framework_security_policies_project_id_fk.rb create mode 100644 db/post_migrate/20240722095914_add_compliance_framework_security_policies_project_id_trigger.rb create mode 100644 db/post_migrate/20240722095915_queue_backfill_compliance_framework_security_policies_project_id.rb create mode 100644 db/post_migrate/20240722095917_index_compliance_framework_security_policies_on_namespace_id.rb create mode 100644 db/post_migrate/20240722095918_add_compliance_framework_security_policies_namespace_id_fk.rb create mode 100644 db/post_migrate/20240722095919_add_compliance_framework_security_policies_namespace_id_trigger.rb create mode 100644 db/post_migrate/20240722095920_queue_backfill_compliance_framework_security_policies_namespace_id.rb create mode 100644 db/post_migrate/20240808125149_queue_deduplicate_lfs_objects_projects.rb create mode 100644 db/schema_migrations/20240722095911 create mode 100644 db/schema_migrations/20240722095912 create mode 100644 db/schema_migrations/20240722095913 create mode 100644 db/schema_migrations/20240722095914 create mode 100644 db/schema_migrations/20240722095915 create mode 100644 db/schema_migrations/20240722095916 create mode 100644 db/schema_migrations/20240722095917 create mode 100644 db/schema_migrations/20240722095918 create mode 100644 db/schema_migrations/20240722095919 create mode 100644 db/schema_migrations/20240722095920 create mode 100644 db/schema_migrations/20240806161731 create mode 100644 db/schema_migrations/20240807103912 create mode 100644 db/schema_migrations/20240808125149 create mode 100644 doc/development/stage_group_observability/img/error_budget_calculation_v17_2.png create mode 100644 doc/user/img/markdown_math_v17_2.png create mode 100644 lib/gitlab/background_migration/backfill_compliance_framework_security_policies_namespace_id.rb create mode 100644 lib/gitlab/background_migration/backfill_compliance_framework_security_policies_project_id.rb create mode 100644 lib/gitlab/background_migration/deduplicate_lfs_objects_projects.rb create mode 100644 spec/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_namespace_id_spec.rb create mode 100644 spec/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_project_id_spec.rb create mode 100644 spec/lib/gitlab/background_migration/deduplicate_lfs_objects_projects_spec.rb create mode 100644 spec/migrations/20240722095915_queue_backfill_compliance_framework_security_policies_project_id_spec.rb create mode 100644 spec/migrations/20240722095920_queue_backfill_compliance_framework_security_policies_namespace_id_spec.rb create mode 100644 spec/migrations/20240808125149_queue_deduplicate_lfs_objects_projects_spec.rb diff --git a/app/assets/javascripts/packages_and_registries/container_registry/explorer/components/details_page/tags_list.vue b/app/assets/javascripts/packages_and_registries/container_registry/explorer/components/details_page/tags_list.vue index aa654a0e3c2..f79f7f6d372 100644 --- a/app/assets/javascripts/packages_and_registries/container_registry/explorer/components/details_page/tags_list.vue +++ b/app/assets/javascripts/packages_and_registries/container_registry/explorer/components/details_page/tags_list.vue @@ -19,6 +19,7 @@ import { REMOVE_TAGS_BUTTON_TITLE, TAGS_LIST_TITLE, GRAPHQL_PAGE_SIZE, + GRAPHQL_PAGE_SIZE_METADATA_ENABLED, FETCH_IMAGES_LIST_ERROR_MESSAGE, NAME_SORT_FIELD, PUBLISHED_SORT_FIELD, @@ -116,10 +117,15 @@ export default { tagsPageInfo() { return this.containerRepository?.tags?.pageInfo; }, + pageSize() { + return this.config.isMetadataDatabaseEnabled + ? GRAPHQL_PAGE_SIZE_METADATA_ENABLED + : GRAPHQL_PAGE_SIZE; + }, queryVariables() { return { id: joinPaths(this.config.gidPrefix, `${this.id}`), - first: GRAPHQL_PAGE_SIZE, + first: this.pageSize, name: this.filters?.name, sort: this.sort, referrers: this.glFeatures.showContainerRegistryTagSignatures, @@ -196,13 +202,13 @@ export default { } }, fetchNextPage() { - this.pageParams = getNextPageParams(this.tagsPageInfo?.endCursor); + this.pageParams = getNextPageParams(this.tagsPageInfo?.endCursor, this.pageSize); }, fetchPreviousPage() { - this.pageParams = getPreviousPageParams(this.tagsPageInfo?.startCursor); + this.pageParams = getPreviousPageParams(this.tagsPageInfo?.startCursor, this.pageSize); }, handleSearchUpdate({ sort, filters, pageInfo }) { - this.pageParams = getPageParams(pageInfo); + this.pageParams = getPageParams(pageInfo, this.pageSize); this.sort = sort; // This takes in account the fact that we will be adding more filters types diff --git a/app/assets/javascripts/packages_and_registries/container_registry/explorer/constants/list.js b/app/assets/javascripts/packages_and_registries/container_registry/explorer/constants/list.js index cbab034d9d4..4698049dd6c 100644 --- a/app/assets/javascripts/packages_and_registries/container_registry/explorer/constants/list.js +++ b/app/assets/javascripts/packages_and_registries/container_registry/explorer/constants/list.js @@ -50,6 +50,7 @@ export const TRACKING_ACTION_CLICK_SHOW_FULL_PATH = 'click_show_full_path'; export const IMAGE_DELETE_SCHEDULED_STATUS = 'DELETE_SCHEDULED'; export const IMAGE_MIGRATING_STATE = 'importing'; export const GRAPHQL_PAGE_SIZE = 10; +export const GRAPHQL_PAGE_SIZE_METADATA_ENABLED = 20; export const SORT_FIELDS = [ { orderBy: 'UPDATED', label: __('Updated') }, diff --git a/app/assets/javascripts/packages_and_registries/container_registry/explorer/pages/list.vue b/app/assets/javascripts/packages_and_registries/container_registry/explorer/pages/list.vue index 2ececeb606e..e58a7920c91 100644 --- a/app/assets/javascripts/packages_and_registries/container_registry/explorer/pages/list.vue +++ b/app/assets/javascripts/packages_and_registries/container_registry/explorer/pages/list.vue @@ -29,6 +29,7 @@ import { EMPTY_RESULT_TITLE, EMPTY_RESULT_MESSAGE, GRAPHQL_PAGE_SIZE, + GRAPHQL_PAGE_SIZE_METADATA_ENABLED, FETCH_IMAGES_LIST_ERROR_MESSAGE, SORT_FIELDS, SETTINGS_TEXT, @@ -159,13 +160,18 @@ export default { graphqlResource() { return this.config.isGroupPage ? WORKSPACE_GROUP : WORKSPACE_PROJECT; }, + pageSize() { + return this.config.isMetadataDatabaseEnabled + ? GRAPHQL_PAGE_SIZE_METADATA_ENABLED + : GRAPHQL_PAGE_SIZE; + }, queryVariables() { return { name: this.name, sort: this.sorting, fullPath: this.config.isGroupPage ? this.config.groupPath : this.config.projectPath, isGroupPage: this.config.isGroupPage, - first: GRAPHQL_PAGE_SIZE, + first: this.pageSize, ...this.pageParams, }; }, @@ -203,17 +209,17 @@ export default { this.itemToDelete = {}; }, fetchNextPage() { - this.pageParams = getNextPageParams(this.pageInfo?.endCursor); + this.pageParams = getNextPageParams(this.pageInfo?.endCursor, this.pageSize); }, fetchPreviousPage() { - this.pageParams = getPreviousPageParams(this.pageInfo?.startCursor); + this.pageParams = getPreviousPageParams(this.pageInfo?.startCursor, this.pageSize); }, startDelete() { this.track('confirm_delete'); this.mutationLoading = true; }, handleSearchUpdate({ sort, filters, pageInfo }) { - this.pageParams = getPageParams(pageInfo); + this.pageParams = getPageParams(pageInfo, this.pageSize); this.sorting = sort; const search = filters.find((i) => i.type === FILTERED_SEARCH_TERM); diff --git a/app/assets/javascripts/packages_and_registries/container_registry/explorer/utils.js b/app/assets/javascripts/packages_and_registries/container_registry/explorer/utils.js index 7ed4ff52b06..87c28e6e9b2 100644 --- a/app/assets/javascripts/packages_and_registries/container_registry/explorer/utils.js +++ b/app/assets/javascripts/packages_and_registries/container_registry/explorer/utils.js @@ -12,24 +12,24 @@ export const timeTilRun = (time) => { return approximateDuration(difference / 1000); }; -export const getNextPageParams = (cursor) => ({ +export const getNextPageParams = (cursor, pageSize = GRAPHQL_PAGE_SIZE) => ({ after: cursor, - first: GRAPHQL_PAGE_SIZE, + first: pageSize, }); -export const getPreviousPageParams = (cursor) => ({ +export const getPreviousPageParams = (cursor, pageSize = GRAPHQL_PAGE_SIZE) => ({ first: null, before: cursor, - last: GRAPHQL_PAGE_SIZE, + last: pageSize, }); -export const getPageParams = (pageInfo = {}) => { +export const getPageParams = (pageInfo = {}, pageSize = GRAPHQL_PAGE_SIZE) => { if (pageInfo.before) { - return getPreviousPageParams(pageInfo.before); + return getPreviousPageParams(pageInfo.before, pageSize); } if (pageInfo.after) { - return getNextPageParams(pageInfo.after); + return getNextPageParams(pageInfo.after, pageSize); } return {}; diff --git a/app/views/projects/empty.html.haml b/app/views/projects/empty.html.haml index 3c37cbb490f..e7d669156c0 100644 --- a/app/views/projects/empty.html.haml +++ b/app/views/projects/empty.html.haml @@ -36,13 +36,13 @@ = _('You can also upload existing files from your computer using the instructions below.') .git-empty.js-git-empty %h5= _('Git global setup') - %pre.code.js-syntax-highlight + %pre.js-syntax-highlight :preserve git config --global user.name "#{h git_user_name}" git config --global user.email "#{h git_user_email}" %h5= _('Create a new repository') - %pre.code.js-syntax-highlight + %pre.js-syntax-highlight :preserve git clone #{ content_tag(:span, default_url_to_repo, class: 'js-clone')} cd #{h @project.path} @@ -55,7 +55,7 @@ git push --set-upstream origin #{h escaped_default_branch_name } %h5= _('Push an existing folder') - %pre.code.js-syntax-highlight + %pre.js-syntax-highlight :preserve cd existing_folder git init --initial-branch=#{h escaped_default_branch_name} @@ -67,7 +67,7 @@ git push --set-upstream origin #{h escaped_default_branch_name } %h5= _('Push an existing Git repository') - %pre.code.js-syntax-highlight + %pre.js-syntax-highlight :preserve cd existing_repo git remote rename origin old-origin diff --git a/db/docs/batched_background_migrations/backfill_compliance_framework_security_policies_namespace_id.yml b/db/docs/batched_background_migrations/backfill_compliance_framework_security_policies_namespace_id.yml new file mode 100644 index 00000000000..9e8ebd3bac4 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_compliance_framework_security_policies_namespace_id.yml @@ -0,0 +1,9 @@ +--- +migration_job_name: BackfillComplianceFrameworkSecurityPoliciesNamespaceId +description: Backfills sharding key `compliance_framework_security_policies.namespace_id` from `security_orchestration_policy_configurations`. +feature_category: security_policy_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/160211 +milestone: '17.3' +queued_migration_version: 20240722095920 +finalize_after: '2024-08-22' +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/batched_background_migrations/backfill_compliance_framework_security_policies_project_id.yml b/db/docs/batched_background_migrations/backfill_compliance_framework_security_policies_project_id.yml new file mode 100644 index 00000000000..9e59b1044ce --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_compliance_framework_security_policies_project_id.yml @@ -0,0 +1,9 @@ +--- +migration_job_name: BackfillComplianceFrameworkSecurityPoliciesProjectId +description: Backfills sharding key `compliance_framework_security_policies.project_id` from `security_orchestration_policy_configurations`. +feature_category: security_policy_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/160211 +milestone: '17.3' +queued_migration_version: 20240722095915 +finalize_after: '2024-08-22' +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/batched_background_migrations/deduplicate_lfs_objects_projects.yml b/db/docs/batched_background_migrations/deduplicate_lfs_objects_projects.yml new file mode 100644 index 00000000000..1db793adeb1 --- /dev/null +++ b/db/docs/batched_background_migrations/deduplicate_lfs_objects_projects.yml @@ -0,0 +1,11 @@ +--- +migration_job_name: DeduplicateLfsObjectsProjects +description: >- + This migration deduplicates lfs_objects_projects by lfs_object_id, project_id and repository_type. + After the migration is finalized, we need to add a unique index on all three columns to ensure + data consistency since the unique validation already exists at the model level. +feature_category: source_code_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/154323 +milestone: '17.3' +queued_migration_version: 20240808125149 +finalize_after: '2024-08-02' # required upgrade stop due date diff --git a/db/docs/compliance_framework_security_policies.yml b/db/docs/compliance_framework_security_policies.yml index 7d7a391580d..b8a42f3af85 100644 --- a/db/docs/compliance_framework_security_policies.yml +++ b/db/docs/compliance_framework_security_policies.yml @@ -27,3 +27,6 @@ desired_sharding_key: table: security_orchestration_policy_configurations sharding_key: namespace_id belongs_to: policy_configuration +desired_sharding_key_migration_job_name: +- BackfillComplianceFrameworkSecurityPoliciesProjectId +- BackfillComplianceFrameworkSecurityPoliciesNamespaceId diff --git a/db/migrate/20240722095911_add_project_id_to_compliance_framework_security_policies.rb b/db/migrate/20240722095911_add_project_id_to_compliance_framework_security_policies.rb new file mode 100644 index 00000000000..22c705dc0b5 --- /dev/null +++ b/db/migrate/20240722095911_add_project_id_to_compliance_framework_security_policies.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddProjectIdToComplianceFrameworkSecurityPolicies < Gitlab::Database::Migration[2.2] + milestone '17.3' + + def change + add_column :compliance_framework_security_policies, :project_id, :bigint + end +end diff --git a/db/migrate/20240722095916_add_namespace_id_to_compliance_framework_security_policies.rb b/db/migrate/20240722095916_add_namespace_id_to_compliance_framework_security_policies.rb new file mode 100644 index 00000000000..28bf19e88d1 --- /dev/null +++ b/db/migrate/20240722095916_add_namespace_id_to_compliance_framework_security_policies.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddNamespaceIdToComplianceFrameworkSecurityPolicies < Gitlab::Database::Migration[2.2] + milestone '17.3' + + def change + add_column :compliance_framework_security_policies, :namespace_id, :bigint + end +end diff --git a/db/migrate/20240806161731_add_index_to_member_roles_on_permissions.rb b/db/migrate/20240806161731_add_index_to_member_roles_on_permissions.rb new file mode 100644 index 00000000000..19381653458 --- /dev/null +++ b/db/migrate/20240806161731_add_index_to_member_roles_on_permissions.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddIndexToMemberRolesOnPermissions < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + milestone '17.3' + + INDEX_NAME = 'index_member_roles_on_permissions' + + def up + add_concurrent_index :member_roles, :permissions, name: INDEX_NAME, using: :gin + end + + def down + remove_concurrent_index_by_name :member_roles, INDEX_NAME + end +end diff --git a/db/migrate/20240807103912_add_vulnerability_count_to_project_statistics_table.rb b/db/migrate/20240807103912_add_vulnerability_count_to_project_statistics_table.rb new file mode 100644 index 00000000000..cf0b4b5b082 --- /dev/null +++ b/db/migrate/20240807103912_add_vulnerability_count_to_project_statistics_table.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddVulnerabilityCountToProjectStatisticsTable < Gitlab::Database::Migration[2.2] + milestone '17.4' + + def change + add_column :project_statistics, :vulnerability_count, :integer, default: 0, null: false + end +end diff --git a/db/post_migrate/20240722095912_index_compliance_framework_security_policies_on_project_id.rb b/db/post_migrate/20240722095912_index_compliance_framework_security_policies_on_project_id.rb new file mode 100644 index 00000000000..7453ea09e9c --- /dev/null +++ b/db/post_migrate/20240722095912_index_compliance_framework_security_policies_on_project_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexComplianceFrameworkSecurityPoliciesOnProjectId < Gitlab::Database::Migration[2.2] + milestone '17.3' + disable_ddl_transaction! + + INDEX_NAME = 'index_compliance_framework_security_policies_on_project_id' + + def up + add_concurrent_index :compliance_framework_security_policies, :project_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :compliance_framework_security_policies, INDEX_NAME + end +end diff --git a/db/post_migrate/20240722095913_add_compliance_framework_security_policies_project_id_fk.rb b/db/post_migrate/20240722095913_add_compliance_framework_security_policies_project_id_fk.rb new file mode 100644 index 00000000000..7f7d720c554 --- /dev/null +++ b/db/post_migrate/20240722095913_add_compliance_framework_security_policies_project_id_fk.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddComplianceFrameworkSecurityPoliciesProjectIdFk < Gitlab::Database::Migration[2.2] + milestone '17.3' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :compliance_framework_security_policies, :projects, column: :project_id, + on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :compliance_framework_security_policies, column: :project_id + end + end +end diff --git a/db/post_migrate/20240722095914_add_compliance_framework_security_policies_project_id_trigger.rb b/db/post_migrate/20240722095914_add_compliance_framework_security_policies_project_id_trigger.rb new file mode 100644 index 00000000000..a8cb8b8dca0 --- /dev/null +++ b/db/post_migrate/20240722095914_add_compliance_framework_security_policies_project_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddComplianceFrameworkSecurityPoliciesProjectIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.3' + + def up + install_sharding_key_assignment_trigger( + table: :compliance_framework_security_policies, + sharding_key: :project_id, + parent_table: :security_orchestration_policy_configurations, + parent_sharding_key: :project_id, + foreign_key: :policy_configuration_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :compliance_framework_security_policies, + sharding_key: :project_id, + parent_table: :security_orchestration_policy_configurations, + parent_sharding_key: :project_id, + foreign_key: :policy_configuration_id + ) + end +end diff --git a/db/post_migrate/20240722095915_queue_backfill_compliance_framework_security_policies_project_id.rb b/db/post_migrate/20240722095915_queue_backfill_compliance_framework_security_policies_project_id.rb new file mode 100644 index 00000000000..39201fee741 --- /dev/null +++ b/db/post_migrate/20240722095915_queue_backfill_compliance_framework_security_policies_project_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillComplianceFrameworkSecurityPoliciesProjectId < Gitlab::Database::Migration[2.2] + milestone '17.3' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillComplianceFrameworkSecurityPoliciesProjectId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :compliance_framework_security_policies, + :id, + :project_id, + :security_orchestration_policy_configurations, + :project_id, + :policy_configuration_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :compliance_framework_security_policies, + :id, + [ + :project_id, + :security_orchestration_policy_configurations, + :project_id, + :policy_configuration_id + ] + ) + end +end diff --git a/db/post_migrate/20240722095917_index_compliance_framework_security_policies_on_namespace_id.rb b/db/post_migrate/20240722095917_index_compliance_framework_security_policies_on_namespace_id.rb new file mode 100644 index 00000000000..3e6621ad7d1 --- /dev/null +++ b/db/post_migrate/20240722095917_index_compliance_framework_security_policies_on_namespace_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexComplianceFrameworkSecurityPoliciesOnNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.3' + disable_ddl_transaction! + + INDEX_NAME = 'index_compliance_framework_security_policies_on_namespace_id' + + def up + add_concurrent_index :compliance_framework_security_policies, :namespace_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :compliance_framework_security_policies, INDEX_NAME + end +end diff --git a/db/post_migrate/20240722095918_add_compliance_framework_security_policies_namespace_id_fk.rb b/db/post_migrate/20240722095918_add_compliance_framework_security_policies_namespace_id_fk.rb new file mode 100644 index 00000000000..5cec8b1988e --- /dev/null +++ b/db/post_migrate/20240722095918_add_compliance_framework_security_policies_namespace_id_fk.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddComplianceFrameworkSecurityPoliciesNamespaceIdFk < Gitlab::Database::Migration[2.2] + milestone '17.3' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :compliance_framework_security_policies, :namespaces, column: :namespace_id, + on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :compliance_framework_security_policies, column: :namespace_id + end + end +end diff --git a/db/post_migrate/20240722095919_add_compliance_framework_security_policies_namespace_id_trigger.rb b/db/post_migrate/20240722095919_add_compliance_framework_security_policies_namespace_id_trigger.rb new file mode 100644 index 00000000000..69d43fb66fb --- /dev/null +++ b/db/post_migrate/20240722095919_add_compliance_framework_security_policies_namespace_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddComplianceFrameworkSecurityPoliciesNamespaceIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.3' + + def up + install_sharding_key_assignment_trigger( + table: :compliance_framework_security_policies, + sharding_key: :namespace_id, + parent_table: :security_orchestration_policy_configurations, + parent_sharding_key: :namespace_id, + foreign_key: :policy_configuration_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :compliance_framework_security_policies, + sharding_key: :namespace_id, + parent_table: :security_orchestration_policy_configurations, + parent_sharding_key: :namespace_id, + foreign_key: :policy_configuration_id + ) + end +end diff --git a/db/post_migrate/20240722095920_queue_backfill_compliance_framework_security_policies_namespace_id.rb b/db/post_migrate/20240722095920_queue_backfill_compliance_framework_security_policies_namespace_id.rb new file mode 100644 index 00000000000..4269e0e791b --- /dev/null +++ b/db/post_migrate/20240722095920_queue_backfill_compliance_framework_security_policies_namespace_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillComplianceFrameworkSecurityPoliciesNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.3' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillComplianceFrameworkSecurityPoliciesNamespaceId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :compliance_framework_security_policies, + :id, + :namespace_id, + :security_orchestration_policy_configurations, + :namespace_id, + :policy_configuration_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :compliance_framework_security_policies, + :id, + [ + :namespace_id, + :security_orchestration_policy_configurations, + :namespace_id, + :policy_configuration_id + ] + ) + end +end diff --git a/db/post_migrate/20240808125149_queue_deduplicate_lfs_objects_projects.rb b/db/post_migrate/20240808125149_queue_deduplicate_lfs_objects_projects.rb new file mode 100644 index 00000000000..399563a843b --- /dev/null +++ b/db/post_migrate/20240808125149_queue_deduplicate_lfs_objects_projects.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class QueueDeduplicateLfsObjectsProjects < Gitlab::Database::Migration[2.2] + milestone '17.3' + + MIGRATION = 'DeduplicateLfsObjectsProjects' + TABLE_NAME = :lfs_objects_projects + DELAY_INTERVAL = 100 + BATCH_SIZE = 10_000 + SUB_BATCH_SIZE = 2_500 + + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :lfs_objects_projects, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :lfs_objects_projects, :id, []) + end +end diff --git a/db/schema_migrations/20240722095911 b/db/schema_migrations/20240722095911 new file mode 100644 index 00000000000..91d134eebdd --- /dev/null +++ b/db/schema_migrations/20240722095911 @@ -0,0 +1 @@ +6a352cd04798df6f147e70e05771e3047bfa52410fb4d08213414d21323d8304 \ No newline at end of file diff --git a/db/schema_migrations/20240722095912 b/db/schema_migrations/20240722095912 new file mode 100644 index 00000000000..06989243410 --- /dev/null +++ b/db/schema_migrations/20240722095912 @@ -0,0 +1 @@ +fe4c99013d813740cd59b0ed3a244d198233b2c1931b727ed751ec9d5ad373f6 \ No newline at end of file diff --git a/db/schema_migrations/20240722095913 b/db/schema_migrations/20240722095913 new file mode 100644 index 00000000000..ded6e6202cf --- /dev/null +++ b/db/schema_migrations/20240722095913 @@ -0,0 +1 @@ +7578b71069ea28c4d392ddaad249bd349ec3593d0a3ea23ed18958611d18e5bc \ No newline at end of file diff --git a/db/schema_migrations/20240722095914 b/db/schema_migrations/20240722095914 new file mode 100644 index 00000000000..4d6ac52128a --- /dev/null +++ b/db/schema_migrations/20240722095914 @@ -0,0 +1 @@ +5b49ae910f52c8bf975f77f8103f7d688c345cc8ee31e74d6417f02e1a819ef2 \ No newline at end of file diff --git a/db/schema_migrations/20240722095915 b/db/schema_migrations/20240722095915 new file mode 100644 index 00000000000..9dd2f77ae2b --- /dev/null +++ b/db/schema_migrations/20240722095915 @@ -0,0 +1 @@ +2c4b6b13a8ae5638792d3756ac9013f3ea822fac4421b887a7176e14e9a48277 \ No newline at end of file diff --git a/db/schema_migrations/20240722095916 b/db/schema_migrations/20240722095916 new file mode 100644 index 00000000000..d61ca823f7b --- /dev/null +++ b/db/schema_migrations/20240722095916 @@ -0,0 +1 @@ +111c47eb2171b12834c880b3dbeb133ba8dc9407da105a4973f791cb4aba9f9a \ No newline at end of file diff --git a/db/schema_migrations/20240722095917 b/db/schema_migrations/20240722095917 new file mode 100644 index 00000000000..c4104a95806 --- /dev/null +++ b/db/schema_migrations/20240722095917 @@ -0,0 +1 @@ +70aa8bbf10488c9e5177abcdb7b0565343758b7f47377cacc0d1db167759574d \ No newline at end of file diff --git a/db/schema_migrations/20240722095918 b/db/schema_migrations/20240722095918 new file mode 100644 index 00000000000..3bda1b55da1 --- /dev/null +++ b/db/schema_migrations/20240722095918 @@ -0,0 +1 @@ +3d76693b6c6b553c2bd681993a1d6e06b7d23d73050fa80294ecf660f2a9d10b \ No newline at end of file diff --git a/db/schema_migrations/20240722095919 b/db/schema_migrations/20240722095919 new file mode 100644 index 00000000000..9788256f47f --- /dev/null +++ b/db/schema_migrations/20240722095919 @@ -0,0 +1 @@ +28409fa04247197472cc18f20ba35c6323cd13c97acd7541a4d1873b71b5b43e \ No newline at end of file diff --git a/db/schema_migrations/20240722095920 b/db/schema_migrations/20240722095920 new file mode 100644 index 00000000000..e50a30a3ddc --- /dev/null +++ b/db/schema_migrations/20240722095920 @@ -0,0 +1 @@ +9f6bfee305d6b1239edbc7e2fdaa1563dcca1c465141582a714a6e2a31333baf \ No newline at end of file diff --git a/db/schema_migrations/20240806161731 b/db/schema_migrations/20240806161731 new file mode 100644 index 00000000000..b64ed385986 --- /dev/null +++ b/db/schema_migrations/20240806161731 @@ -0,0 +1 @@ +b1fbfb48e36e0fce50c27fec63cf83ce9c78a317a426733ade3e6863291e9d6a \ No newline at end of file diff --git a/db/schema_migrations/20240807103912 b/db/schema_migrations/20240807103912 new file mode 100644 index 00000000000..c2d984fcd08 --- /dev/null +++ b/db/schema_migrations/20240807103912 @@ -0,0 +1 @@ +c51b3cb6d6e9f0cdb2169322bb94641a2f8fabb2f4343b44af2905ceb2a4bcee \ No newline at end of file diff --git a/db/schema_migrations/20240808125149 b/db/schema_migrations/20240808125149 new file mode 100644 index 00000000000..98be0278225 --- /dev/null +++ b/db/schema_migrations/20240808125149 @@ -0,0 +1 @@ +b484b543db35e018c2a9bb9bfa3d0ff65f8f0ad6874509841660275ccef6c855 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 524097ded5b..b3e6b7a9892 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1330,6 +1330,22 @@ RETURN NEW; END $$; +CREATE FUNCTION trigger_6bf50b363152() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."project_id" IS NULL THEN + SELECT "project_id" + INTO NEW."project_id" + FROM "security_orchestration_policy_configurations" + WHERE "security_orchestration_policy_configurations"."id" = NEW."policy_configuration_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_6c38ba395cc1() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -1362,6 +1378,22 @@ RETURN NEW; END $$; +CREATE FUNCTION trigger_70d3f0bba1de() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."namespace_id" IS NULL THEN + SELECT "namespace_id" + INTO NEW."namespace_id" + FROM "security_orchestration_policy_configurations" + WHERE "security_orchestration_policy_configurations"."id" = NEW."policy_configuration_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_77d9fbad5b12() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -8994,7 +9026,9 @@ CREATE TABLE compliance_framework_security_policies ( policy_configuration_id bigint NOT NULL, created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, - policy_index smallint NOT NULL + policy_index smallint NOT NULL, + project_id bigint, + namespace_id bigint ); CREATE SEQUENCE compliance_framework_security_policies_id_seq @@ -16395,7 +16429,8 @@ CREATE TABLE project_statistics ( container_registry_size bigint DEFAULT 0 NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, updated_at timestamp with time zone DEFAULT now() NOT NULL, - root_namespace_id bigint + root_namespace_id bigint, + vulnerability_count integer DEFAULT 0 NOT NULL ); CREATE SEQUENCE project_statistics_id_seq @@ -27249,6 +27284,10 @@ CREATE UNIQUE INDEX index_commit_user_mentions_on_note_id ON commit_user_mention CREATE INDEX index_compliance_checks_on_namespace_id ON compliance_checks USING btree (namespace_id); +CREATE INDEX index_compliance_framework_security_policies_on_namespace_id ON compliance_framework_security_policies USING btree (namespace_id); + +CREATE INDEX index_compliance_framework_security_policies_on_project_id ON compliance_framework_security_policies USING btree (project_id); + CREATE INDEX index_compliance_frameworks_id_where_frameworks_not_null ON compliance_management_frameworks USING btree (id) WHERE (pipeline_configuration_full_path IS NOT NULL); CREATE INDEX index_compliance_management_frameworks_on_name_trigram ON compliance_management_frameworks USING gin (name gin_trgm_ops); @@ -28215,6 +28254,8 @@ CREATE UNIQUE INDEX index_member_roles_on_namespace_id_name_unique ON member_rol CREATE INDEX index_member_roles_on_occupies_seat ON member_roles USING btree (occupies_seat); +CREATE INDEX index_member_roles_on_permissions ON member_roles USING gin (permissions); + CREATE INDEX index_members_on_access_level ON members USING btree (access_level); CREATE INDEX index_members_on_expires_at ON members USING btree (expires_at); @@ -32205,10 +32246,14 @@ CREATE TRIGGER trigger_664594a3d0a7 BEFORE INSERT OR UPDATE ON merge_request_use CREATE TRIGGER trigger_68435a54ee2b BEFORE INSERT OR UPDATE ON packages_debian_project_architectures FOR EACH ROW EXECUTE FUNCTION trigger_68435a54ee2b(); +CREATE TRIGGER trigger_6bf50b363152 BEFORE INSERT OR UPDATE ON compliance_framework_security_policies FOR EACH ROW EXECUTE FUNCTION trigger_6bf50b363152(); + CREATE TRIGGER trigger_6c38ba395cc1 BEFORE INSERT OR UPDATE ON error_tracking_error_events FOR EACH ROW EXECUTE FUNCTION trigger_6c38ba395cc1(); CREATE TRIGGER trigger_6cdea9559242 BEFORE INSERT OR UPDATE ON issue_links FOR EACH ROW EXECUTE FUNCTION trigger_6cdea9559242(); +CREATE TRIGGER trigger_70d3f0bba1de BEFORE INSERT OR UPDATE ON compliance_framework_security_policies FOR EACH ROW EXECUTE FUNCTION trigger_70d3f0bba1de(); + CREATE TRIGGER trigger_77d9fbad5b12 BEFORE INSERT OR UPDATE ON packages_debian_project_distribution_keys FOR EACH ROW EXECUTE FUNCTION trigger_77d9fbad5b12(); CREATE TRIGGER trigger_7a8b08eed782 BEFORE INSERT OR UPDATE ON boards_epic_board_positions FOR EACH ROW EXECUTE FUNCTION trigger_7a8b08eed782(); @@ -32722,6 +32767,9 @@ ALTER TABLE ONLY release_links ALTER TABLE ONLY bulk_import_export_uploads ADD CONSTRAINT fk_3cbf0b9a2e FOREIGN KEY (batch_id) REFERENCES bulk_import_export_batches(id) ON DELETE CASCADE; +ALTER TABLE ONLY compliance_framework_security_policies + ADD CONSTRAINT fk_3ce58167f1 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY ci_pipelines ADD CONSTRAINT fk_3d34ab2e06 FOREIGN KEY (pipeline_schedule_id) REFERENCES ci_pipeline_schedules(id) ON DELETE SET NULL; @@ -32944,6 +32992,9 @@ ALTER TABLE ONLY projects ALTER TABLE ONLY dast_profile_schedules ADD CONSTRAINT fk_6cca0d8800 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; +ALTER TABLE ONLY compliance_framework_security_policies + ADD CONSTRAINT fk_6d3bd0c9f1 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY vulnerability_merge_request_links ADD CONSTRAINT fk_6d7aa8796e FOREIGN KEY (merge_request_id) REFERENCES merge_requests(id) ON DELETE CASCADE; diff --git a/doc/development/stage_group_observability/img/error_budget_calculation_v17_2.png b/doc/development/stage_group_observability/img/error_budget_calculation_v17_2.png new file mode 100644 index 0000000000000000000000000000000000000000..66468e29ef179a70b5d390b1eb7dcb9d58ded136 GIT binary patch literal 10789 zcmbW7Ra6{Nv$k=9C%BW~?(XjH?(P!YB@isY;1=9H=-}?|3^F(b&)|MI-#Pcc)J4nc z)oWMnRaO1&r+2ifvJ4tBAu<#c6q=l@q&gH7eD}w7)+gAHqgrq6)W@k;RZ&yw{r!D? zeZ$Pu3IIs0tfyLl=os6ISA zXsGK%Ma6h~`&(Mrh>A!yH?{Wm_Se+Z?dJK0lZfMQ{x73uZI){ys|yluqI7*Hji%esa`E?MbCylx*0x zV=x)O>YdXAOR9hOG(FD`5RVh67=(bQ|9~{X_q03@vyBFUkMg8v2 zmUP32J2+J>e0!aID5?lcwCCu3nsFgD90kiM@=nHDX{gQhoOgF5#v{WOt&o@*-0GD{i#X+ z(fr(6zmdBlIL6W_a@^JFt6m5UaQ&fQo8>-8TMc>A(=0wx5oiN~_yCiWFE*R4=y$%t zd}J5ry;H|DX#Uewpgh??8xL!-`zp59b_e9D^D(Tr^OG%>tL@vh=>YU4Z@EP?XFRKf zKL*^n9iS;IYwEH|_NtdU{SL=;j=J~sRuQXn`meTRmrdmSOdwGF3w;}dFn2~u0~ z2KMaJc=sW9ug0?4BUBUc?UOB`yYloSWY#z#@C{JHZ&JN`RGDXR63|ly)76}fu zTwq$YqL{x3u{iZZITlpVZAPf4uMqn(hB&p@A*y5oEM+0JCXC9$nPUh<`x@xDLQ+k6 z*verTVss7;X~CgusOv>(+A5vJi_#+D=PBa+TQG{V#EyzTIk75{s?;J=#4oCiiYBE+ zS?A^x_7eTep&~N9e4V{UyyVk!5c`$mw2=O34N=2LZSVHK{aSmmqIr|;FM{ga z3Kp=%oV8E>v(~$fi?iHCbaQr5h-jolBv;Z}q#d7BG{!4n6<317eTY-}oYgtGdneXQ zFAfkdx%6E~%2l01Hs4a7yCaiMGAXK5UBKG~gssXFHNj91%;+5$8sK$y!>MN>o@ve# zN6scnP-*!iiHJbRYu7*I(khCP^6e{RI;0hEW_vOx=&YbIQkP>dGqBV;FTx|Oa9{?m z3%%{0Ny8oX7@?t9T~Y+-sK%(h3_4Wn+art%9zWAOXa$G1q+2HNv#LdV2%gCTNfMJP zrd_bjtb2lZ<&@7ZC$40-@Jm~!a#;^=S-|O&m&mC&&}TCYaz13dCSlWEFQ=T+@eUvX zvLTl9+RuoUY!(;JKzGp1!%baHN=)$5M|7Uaw&015$U_@|olu9J@jnfC|2^O=P00@? z_R#N}S)AIB$%RjxtTZvaK|W|U@CxQzGGJA{glS}g6yCY#$q@-BBsvYq(ubULmJ<)6 zxv-ALOg4?Ci|w_qqev6bwsqOHK$EUdvw84nd&cT!$p@Yqi5%+{M2|K~_o2w1o?^2; zr#v(F)2FDQ-?zN$)PQblk8XQlv&k->ruX zb$60rF$4q4arL!1eY#B(5bi*5<+Yov9`vMoygpWS!F`}x{DG6|L`k9PpVYbO& z!4t+n5Ub^PzI7YZ{;hgl#s#l_`ZuCZB(TEd<3 z?l{V-g*OyY(8J&MG`!rca=xrWWs}ZLHc%I7Rubb_r2@1ILQ1VzNB)bivVJkKrCiQp3 zTzYg^^nJTF)k2>>q1HG*Bp`X}r!-^ij5mU74f7?84~;A9oCf3d(K7Z zkGgiZtfrswbIYbdFFtely%UcuZQEQA-tOj=qn@P|Fc9PtVV-#%s+G1aExs;;B>2CX z5q7RLXrAeze&$mqmAU`P#`DbIoNBQVO@v;4QfQOvDK1r^=iJVF@z%+e)tcm{@VriF zF`A3J%Qf3uUd63~CP5gw^Wvw$%=bpuBRgxXA8F{$QHf{fefKS?R;sLf-R`()3833? zFm^s{RBv~od}|NuK7XXFelb5aUv4>Yn;WY?0)YR{ytWrwL!94vSGPtk|E;E=iD41c zV|rf4SW+^%!uLSnfB_Tc5aW(w$^89e_Kv#1X^R|~)Tf`+{#k<&asHI)5wl>kTt0Pc zFvR~b+~2b%rIj7ar9?JXe*4n&@89rX$1Zx7fX|;SZp^%Yc#gr*KxFQR)vLD54hoLC zGgAtRugz0j`rKaP`)pJgJ_^N$H9eAL+aq80x8f(?#M3bsH3tW4Xb0r+NNuLUU4L97 z3(awk9U*Ckg;!6ET%J18e6To~IST(N=ezb)VblN=E%bl-&E+*xol$m$%Vk0?wCBcH z>RJ>p=(FMEI&ZcoFzaD_^kO5J4aS@NhsMI34QW`?Qc?-(YHrjY!-!{-4|D~l>CPc0 zYR=i}mvn3W`wY+0mER=f&lR4F4xevos|sYPb_S>lEhHr+l(T$ZVyh@XdsVnw1s|*H z5CGtBVD97Z09m<2W0m;UMKT7@CrgR!aVa9oL0cd#(R{~iw3B3%+_c#GJGkBj^48@8 zn8uYH&etOz&6Dz>@vcO~teqUaos8qgB6Wq2Y9=5!+4|SmQ5hm-dIh}R9S^h!?rA|) zLrrb#Nywlc_p?IS$hi;$!6?w}6){!`(dl>TMeXz;D9#-f?iXcKc{h=2uTM@-(WP3Xr&F|6b*%)vaaSF@ z8o9=o^5qot7)XHpDj23Jy*va*xt<7YtdCFeX4o|TBAivh?E9vR)e806M0C{a6HQ)0 z?!%QQy@dC(5?S}l-2rBdC)RQA{qtfOOyapKgbg; zy_V{WWBwUSqki{r_V@!oo>!pDB5H-9W5%&HeO4JR7{gtY!IX%ZnWaq;%N@?~v1;pxQq`Eys3hfBZrs%X#^a}1ir*+5{YMs~b z&@%!yfIfWh03qnMt26vQW|9D{~q)DnqPW_E& zvv$o{C&E()%RStMr_YJlmLlKc*p(x)=x6?jRahP_sg6y6w)zPDCXCM=BTnn8JZvUf z>NCXfQ*~In#ynbCw-ztvBtl;-*7bKw+>Le|>rPgSw?1V$P1rTP+k1*YTr;Bklwfc>y4g68?vf^h(I*_!{4=`snHR)Yek+DYc zBX>bdS|iWA?%D7KOe}7@{(jnm)}sp`>PhCN42*Crvv7dV`UVsj=<3w!cA~j(E|+4rOaXv-`?RDCU7zfZz~3|Wg0KkQiaHc$F=-;$L!HPWm(ZB1ff_Ed4% z8npP_ZIsw3Tkfp6qOGp=>lSi_vEof#IXark+6=Tza94~~*^#d9h#Rfdw&%Ae>piX0 z>o~!=*Vp^dpgMb2!u-;MQ$x&)cIh>nKBv5`1YoGsJr;-#>(wPQZS%6_LmPl&sdBOP z&qe>9uDB#8M}Ibv*ti{HvG~yXgs}r{VI6P75Zw2Jd(4F&`d5`zzFAMoz7Ac<1WRq) zFBsJOqp!Z~t}zADhdDrG8EOq@D|D~V1BQ9OCR_Yf^N*hTVq8@Gk%}((-@)iotpN7M z`}xRe3QGY31FECmdHfi7ORWWKwb)t3&k!G2g)S5#&@4ZTIlq(_P13R>9iS$^rev>& zD6cF0N1;qW5IDjvborE1(%sz!tOjgr?dExWcq)A=x>C z>It%P=uvZCxBlF<-06mB7VwH%H!!4>PVguoI-RzQAE^9Pf>F!Re+fV-h=Q{6V5 zVL;V}*ZF=e6Y>5%X0p1+6YLvd{Yg{-$GV4v&WM(I%WfKP4&QQvONcZ)J-|E1V!rDi z^!k;y0ovF_Vsb!;n5_-`4Kw7&SWl}L)CxA_P1xI-IhwDb0wp0_^dhh3hpy|a-O zz%v9&m3MQfQK1Nq!d!<5AxsovYZA}o;%NK`*BX7xYG!$Lo=?7FelX)U5oV&wrt_keU(5&Ke5R75e-5?U1%@`8^=3rSU)!D`rD`_9s8N+6j=d|OKx?NvA?p5r!|tY8^hnH z%}S3AkD!L}5h#CattJDO;Vc1!h49}84nFzI#UyJG&MzoUh&BOev_zP@XUwpvcS~Q{ z6g|iQ+}}y547B5!P*|Ovtc^|mSwZjuPebzvmE_4fPy^cDzmGaU#XdiTX z^2(y`k7HOPB<}mFu^a@el}RipXYX1=HX0#tUf8TK6qr-(FZsQzJTRQEa|Y3C5e+Bx zvPA4(u6=m=0QdearykPvfc3=;30at*zcnD?x4`+*f8$hF@oUV%^}0n?oS*Di z_Dk)h;o`SdiumVxp-T8)!6hB8BDy9jbghKfnBSKnNX#N6`bEfwdLlOGNchFSkI`Q5 z^PVv`UCX3&pSe0631jUdLg`BEmnx+R<}1(~69C^y8&)ngvk_5KiR$jk?3{3@MlLC-d4|l`P*6Le6Q}94TsWaR(zJ$GQZ*p3qW%tb zPy%&ZCk_3=F*WYWrX2$-t9$w9BbbLTwdZp(oQSFExxcv(Yb~G3Hob8) zqxAo7zf(o}`P;yb+0>cUL3N$A!;aV`0I1R%DtR?0?;6I9QSwU7+F9?iVZ>y)LdWB>34xq5`i2qQVwLWbVTJne?=V5(a=_jcIz)-gcvsLzAQ7Z zaHDVLN}4me>Z0A(n*LZw8T$IqxOv^euO}1k?u*ck8;4dI!wpZ}P4+ju*duygO>K8e zG@-}W!DFR`0Q}#)BY2qy?n+3yE-E~9m+X-_$y^-MuhM-tp3a#B3fR#bXE!FCm&=9q z?$LWc&b$07)o3j--^Mmuy(g6~{YL2;;?KQs(>_!LUA8f8lh$MZ#og;f-ye!ye?9KE zU=nYGnxiS3gU?|sA*Kr;nxz*cw;$vvogn@KRb95FFaX~^IRjPBFdN<;I1PRs8_XB z4q5~0_tks0_qq;tVm~=cCGbf})n0TYsES$Fz7Mo^G91ngylueOVlJiZWYr z_83;1@1K0!Sr>Skn?m($?;mCHSd(k>$$#zVer6U*PR?MFil2NZ;M{5e)Ex~OU?2L< z@S6M9PV}fWJ(Hs2EAE4L^P1cd&i3nM;2GX$YF^{L6%SC3>5JkXVcuk9O1XlfzL^XM z)@`Yp81dFN3I)yV|B*Nzclr7bNd5F=I^H8C$uOXyNnHIxwJc2Bv!@*-f)QM3VNnb1 zZ8@x&fA(H>awBqydHZ^VSEwR4uJb3V_9gf$%EpHSiYScHkNWHt?5nR13g*tCI!R|Z zt5d;#D#6fAwJ4Ah#vb;WRpTixS>w8W7n(UA=&jmp2n$y7!xN{CqQQU#4I3cBZyur6 zI!}qi>%ajhkv)c&G$QlnCGRtN0H(4QI#4t3&hYxTzIj;NMvG1*eUrT}T18Xk`#6G) z>z%Fi&n@ZEerW0V7{WD)J2X}L;53)mll#`+d^gni(lNMr{3_uQ7adJt>_)-H_KtN! zX8^&7{GQIbQ~W>LosuCE zGT*CJ62!#z;Rl)*ujYRG-)+^z3=t{3sGJu%h}@ z!R2q)-YUi|%*`uR0kq?ZPqdZ{6Fnhc2m*)XKmF@1l4M1|6wwOTmkInBKPv47?!#RT zpX5*0qHnEHj_{@=A6{DYva?o^(R<$4-+8 z8AyCk$DMTUJLNtSmTK5KWaP7$=hV@oY67Cb zcPiZPP6wnZm79FhEe)ar}T3~M&RiOll?t6ajIIZiVk zr#oZ_9$}uVR2Nl;Wi}S#{B4oRolf&TnwvQc6mUc_Q;XtT+A)LOV9It*tsbf0Li*N+eh;pPimz>V$GzyF ztAig&Lt4J1zd>aj){rJn8rQ<-e_f;O)!4!VND0_l67*_wac<&n9o!SC-Ef688T5V1 z$TBu5ZVtP?M<}0!Se3=$;4T0KbLFg&U7ORaSGv5IkS&}$`uS$^s6v7G{>0MhsjSz( zh0_)GSvoFhuyMdB(?+399dd05LM6}(hcAimGN-g0@e)%p>ia$k5_0!k>N6YNP~lFa zxF1QwnZZAfM39fKwQaQ;v;c)Jq2^cy6`u1Esy9%qU~1!A4^#)J2il2Y(;Iz9EPO;7 zar=oZ!cj|>=K|_v+AcZ}-Z=l38{T{Mlu5R%dw}g$=syJaq$3ks-_2Nt=*k_dVYP+x zP*i@evL{NuB~HkD!%Cj%3%{x{r@XCR>i6akKpm2WaW|XE;t0S>64qC{-dhA5nV_KN zIEM}}6pNeV;tT$VRby7d$IA!1L8eW$qNb;4mHcSui}-)crPx)Li(Q0aQq~nTZ?z{r z9$l=@s`~J+rR~^nU5Zt_c1L;=+P}i~db_O#w*AWTEHLbJ7JIxaB#%eCg0xgLer|Ad zhRQCkYOH6`c}BoNlv7B;?Erd_(l$<12$t= zZ(+%wV=5lNP;dhf&T3kf5V#|+8M?M>%CUx&5jr1_!W%|mdsPh;SCDEZ5>pBEyGXMAwxxMVln6`oq_GBIIbByimopOs=WS=%lOo#cPn#ZTL8V*LPVa5fWUh7>6 zUSN@Nnxj4^R{^xc4#FGeIKzKt*Lz0bWvs=VL*f2YDJielk|E@&1zC8X-0z`Yqqx&j z&9i?B`>x?(IqO%bMLn7RprU?7L9BCp$qZJ^d2&YVdszI8h=zQG-dEl$ZiN>|7U{?^ zJKUYngOx;%W*547ljj_>@hHXk(ra%Nx;4lLQpK_pF?(|OcVr-(T+{ZX6cq3^lB&G0x$;(A#I&K z^#?8eNZhdCT72gw%yqC)8j^_cm0 z3rtQSveqX|J@aQH+K`iYHP0tCxGtka0YAonEa^N^D+a`fNegg)Wgno?x00jGH>v8|&Cc@syo}we=om2E2G&$5-{x zq^sl5;|}jxOr%t{XeCAk%V`-I@60x&PssGUW&?Yf61JpxAR*c}R!afenSAlJsf|E= zW3;{a039jhkb^Q8hWj5&O7s3#7q@*i!#Gkir|zL^RePcYKmD9{czc&%%zCD7$R_ID z4IAj$;M_x>hu?ynfRCXmikiEfM?*0`R%|Nzwkp1oVRE#cC8{~uuh*+Pq%Lx=*siT2 zC5<@v3R`b7u!`t}#iSQBjv2Uz>1m$H;o%hh^wj+3g9bueF5^0em95m}VXDk_K46bT zi0wsiHI?x41G_F6>AUJ*A`8_so^lvQwfQ!(R}cQ*kv?QXh&a zj!Gni)pU030SE97y8;($4_Q#a7Q}dEF*EOVW0SG zr`&D&y#vyxlUPZ3mp+m1aoV|$KM?9G*5*+3R0hfFisn`E z5_C*))%S&5eNL;o?(AD)Ti~fG3*nkS!J;0vAC760QbfyWo15=0wjvLonKF>(d&AC$&kN_R984NP~2^mfwkXt9L-%%*EQkegq1o#UCgXT5{Q)k0PDS`Db zs=_73!;|P1Te}!(BZ`wfw*H5Q+L9b|Kh#0~VF6l-lo_YJrz-tR0wz{bWZ`358fWLj zw7!Q-$GF?HjCpQdTXrm>Z)hbao*uAwS5WRBH(=ZDqEY`yiqTg=PAgt@h_$X`>u`?A ztD!XUT66cFs$*5|QLvhGMo7-tOxAzR0Zr;AHA~VuWE{w{|2XHy!6joVC98M@*%9<_ zg#Dxbz)cD}aphP<4;mk6J>sAmbxfLkG`SEA91g5{cd0kd0Kj^Q6@GKmmN;-E=f*AV z`Q>*y0U!6ZUc!w375{R$oZ6hN^>n zh*fE{q2^M8d7)uFqI_RsD}(lV1ol4eBwTnNOAmB%?m+LbRlbQOyJ1dv%b6I>4RVUI z>X}z5&Aqk9Y=EVQk0uhPAROqs*3{PJQK4R5eGbVDYDnC6(VHA+_4w7uN@vJ zJLOI96_kQQL?EVTO?d#5Z1EM`9Y#Y@J(>%yKyNH5Ut z-C)BoF5xz%dN(sSN8VE1xIX+f99rH0fFR7p{`@Dog$C9>QOYQBXn8}8iz19PG(8wG zQ9?65mGL1N_fmj`|3?AL=vwOA)d;bO#14nF6O_Pi8 z5#`yn{SE0=lLV=f_KuQNg7&WSl>}*qocyp zrNDI4Zvx}V98?>-pDvezU+M01zjxe1uYWeu;2|@u7uIQf(j7H$Ns;CKQF*b#olaX} zSA2P=shbp8$}SCPkn3pr2-K%i#rf8ZXMpGHWQojdUF7X-r zG#EZ-yZhkI-W=E;8S3~4rlInFe07Pp+RDYuGrZtt#D=C)jyUHXz7Ukh&?hc*FwUf! zhTd)a)I(eCX*d{@Kz-LrCFdMLUEtH^{J3;a$tzJdUooeX;T6v~BJr zWE`H-G|{h+2RmsUnwSu0oDkdTw^R;WsgxDxmMN>gDB?VLQ%q=hr&KrUM{Ud`Mwt|Z zaHEq6kT~ss@lYcHPhTN(a9s!+NC0Dyy2aa6{!kHPXTMe1Yjy@?tKn_^8{##xFpJib z9Q$TOBMS)j-meTph~?K)`&X$VtLUj0So(MN$5iUGgFi-BtC|xOPb3(Pqj{j@Nxbz{ z8Hwu*M7uO!j|HK--wfF@v?vlzVlwJkD&Rn`0K$z)Bh|n(qcgOkOdz7WXX0u;y`rxQ zBq_u+(W@#s89SaR1aGpw{#+>6pE@%B%Zwihg2I2_xT1aMh>#X6w!N^t>5CkwYDx-XYkQ8)_a{P7XNUzFN#{ zv*C8d&vI_ak*rrY5I{9>dgZ7MxzHz`ghn3=yL`^kIIrn~IqO_ML`F|deZWW5i416| z-H=6TDf|>oV7NuP8Q=l60fe^nFM?iet_IC+;H103uby6}nx47M-dQ~6Jf{ie#}5uw z?y9n_n{l1~FL0uZ`~XgK;7){2f?Z7QV$WW)t2T5_Oed5}DWkPs$RB9s|7A^HV;&e$ zPu8^WZsz8oM=Nn`zXeXRNO#K&w#LG?smwGDY8O9eQzKTJ+LfofU)^m(Y4=-3e4F0x zEbJ2pnr)=ayR(x_SR+qS0@rH_@AmGy$_=wcx&1}!OnQnb zHgaYJ{SDGRQ(9(27?#%Q5MgJU)s+9z?>K?h0RItb@7SYsVD-t2LQIOAbxBJfB4M=Z zcUjHAs$UWa8B-+w|ID&i5}I+4r^eo!y%7naCDe$cFs!_WG)-BhvkasGpH5#*801~? zGj>4tLcO76Q14h#&|oUx__JWrz5H&*<9_CiE-^2>uEO z*xL?7TLAiPDZapezb7Z9ELkgV8unkvfduvd literal 0 HcmV?d00001 diff --git a/doc/development/stage_group_observability/index.md b/doc/development/stage_group_observability/index.md index 057a2ecc092..4d04d044394 100644 --- a/doc/development/stage_group_observability/index.md +++ b/doc/development/stage_group_observability/index.md @@ -87,9 +87,16 @@ component can have two indicators: The calculation of the ratio happens as follows: +![error budget calculation](img/error_budget_calculation_v17_2.png) + + ## Check where budget is being spent diff --git a/doc/user/application_security/dast/authentication.md b/doc/user/application_security/dast/authentication.md index 3bc2675d2b1..c9e6e471649 100644 --- a/doc/user/application_security/dast/authentication.md +++ b/doc/user/application_security/dast/authentication.md @@ -70,25 +70,23 @@ Use the following CI/CD variables to configure the authentication actions requir | CI/CD variable | Type | Description | |:------------------------------------|:------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `DAST_AUTH_COOKIES` | string | Set to a comma-separated list of cookie names to specify which cookies are used for authentication. | +| `DAST_AUTH_COOKIE_NAMES` | string | Set to a comma-separated list of cookie names to specify which cookies are used for authentication. | | `DAST_AUTH_REPORT` | boolean | Set to `true` to generate a report detailing steps taken during the authentication process. You must also define `gl-dast-debug-auth-report.html` as a CI job artifact to be able to access the generated report. The report's content aids when debugging authentication failures. | | `DAST_AUTH_TYPE` 1 | string | The authentication type to use. Example: `basic-digest`. | | `DAST_AUTH_URL` | URL | The URL of the page containing the login form on the target website. `DAST_USERNAME` and `DAST_PASSWORD` are submitted with the login form to create an authenticated scan. Example: `https://login.example.com`. | -| `DAST_AUTH_VERIFICATION_LOGIN_FORM` | boolean | Verifies successful authentication by checking for the absence of a login form after the login form has been submitted. | -| `DAST_AUTH_VERIFICATION_SELECTOR` | [selector](#finding-an-elements-selector) | A selector describing an element whose presence is used to determine if authentication has succeeded after the login form is submitted. Example: `css:.user-photo`. | -| `DAST_AUTH_VERIFICATION_URL` | URL | A URL that is compared to the URL in the browser to determine if authentication has succeeded after the login form is submitted. Example: `"https://example.com/loggedin_page"`. | -| `DAST_BROWSER_PATH_TO_LOGIN_FORM` | [selector](#finding-an-elements-selector) | A comma-separated list of selectors representing elements to click on prior to entering the `DAST_USERNAME` and `DAST_PASSWORD` into the login form. Example: `"css:.navigation-menu,css:.login-menu-item"`. | -| `DAST_EXCLUDE_URLS` | URLs | The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. | -| `DAST_FIRST_SUBMIT_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element that is clicked on to submit the username form of a multi-page login process. For example, `css:button[type='user-submit']`. | -| `DAST_PASSWORD` | string | The password to authenticate to in the website. Example: `P@55w0rd!` | -| `DAST_PASSWORD_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element used to enter the password on the login form. Example: `id:password` | -| `DAST_SUBMIT_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element clicked on to submit the login form for a single-page login form, or the password form for a multi-page login form. For example, `css:button[type='submit']`. | -| `DAST_USERNAME` | string | The username to authenticate to in the website. Example: `admin` | -| `DAST_USERNAME_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element used to enter the username on the login form. Example: `name:username` | +| `DAST_AUTH_SUCCESS_IF_NO_LOGIN_FORM` | boolean | Verifies successful authentication by checking for the absence of a login form after the login form has been submitted. This success check is enabled by default. | +| `DAST_AUTH_SUCCESS_IF_ELEMENT_FOUND` | [selector](#finding-an-elements-selector) | A selector describing an element whose presence is used to determine if authentication has succeeded after the login form is submitted. Example: `css:.user-photo`. | +| `DAST_AUTH_SUCCESS_IF_AT_URL` | URL | A URL that is compared to the URL in the browser to determine if authentication has succeeded after the login form is submitted. Example: `"https://example.com/loggedin_page"`. | +| `DAST_AUTH_BEFORE_LOGIN_ACTIONS` | [selector](#finding-an-elements-selector) | A comma-separated list of selectors representing elements to click on prior to entering the `DAST_USERNAME` and `DAST_PASSWORD` into the login form. Example: `"css:.navigation-menu,css:.login-menu-item"`. | +| `DAST_SCOPE_EXCLUDE_URLS` | URLs | The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. | +| `DAST_AUTH_FIRST_SUBMIT_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element that is clicked on to submit the username form of a multi-page login process. For example, `css:button[type='user-submit']`. | +| `DAST_AUTH_PASSWORD` | string | The password to authenticate to in the website. Example: `P@55w0rd!` | +| `DAST_AUTH_PASSWORD_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element used to enter the password on the login form. Example: `id:password` | +| `DAST_AUTH_SUBMIT_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element clicked on to submit the login form for a single-page login form, or the password form for a multi-page login form. For example, `css:button[type='submit']`. | +| `DAST_AUTH_USERNAME` | string | The username to authenticate to in the website. Example: `admin` | +| `DAST_AUTH_USERNAME_FIELD` | [selector](#finding-an-elements-selector) | A selector describing the element used to enter the username on the login form. Example: `name:username` | | `DAST_AUTH_DISABLE_CLEAR_FIELDS` | boolean | Disables clearing of username and password fields before attempting manual login. Set to `false` by default. | -| `DAST_AFTER_LOGIN_ACTIONS` | string | Comma separated list of actions to be run after login but before login verification. Currently supports "click" actions. Example: `click(on=id:change_to_bar_graph),click(on=css:input[name=username])` | - -1. Not available to proxy-based scans. +| `DAST_AUTH_AFTER_LOGIN_ACTIONS` | string | Comma separated list of actions to be run after login but before login verification. Currently supports "click" actions. Example: `click(on=id:change_to_bar_graph),click(on=css:input[name=username])` | ### Update the target website diff --git a/doc/user/application_security/dast/authentication_troubleshooting.md b/doc/user/application_security/dast/authentication_troubleshooting.md index 54b9d743abd..ec3f42876d9 100644 --- a/doc/user/application_security/dast/authentication_troubleshooting.md +++ b/doc/user/application_security/dast/authentication_troubleshooting.md @@ -76,7 +76,7 @@ Suggested actions: - Check the target application authentication is deployed and running. - Check the `DAST_AUTH_URL` is correct. - Check the GitLab Runner can access the `DAST_AUTH_URL`. -- Check the `DAST_BROWSER_PATH_TO_LOGIN_FORM` is valid if used. +- Check the `DAST_AUTH_BEFORE_LOGIN_ACTIONS` is valid if used. ### Scan doesn't crawl authenticated pages @@ -92,7 +92,7 @@ Suggested actions: - Generate the [authentication report](#configure-the-authentication-report) and look at the screenshot from the `Login submit` to verify that the login worked as expected. - Verify the logged authentication tokens are those used by your application. -- If using cookies to store authentication tokens, set the names of the authentication token cookies using `DAST_AUTH_COOKIES`. +- If using cookies to store authentication tokens, set the names of the authentication token cookies using `DAST_AUTH_COOKIE_NAMES`. ### Unable to find elements with selector @@ -105,7 +105,7 @@ DAST failed to find the username, password, first submit button, or submit butto Suggested actions: - Generate the [authentication report](#configure-the-authentication-report) to use the screenshot from the `Login page` to verify that the page loaded correctly. -- Load the login page in a browser and verify the [selectors](authentication.md#finding-an-elements-selector) configured in `DAST_USERNAME_FIELD`, `DAST_PASSWORD_FIELD`, `DAST_FIRST_SUBMIT_FIELD`, and `DAST_SUBMIT_FIELD` are correct. +- Load the login page in a browser and verify the [selectors](authentication.md#finding-an-elements-selector) configured in `DAST_AUTH_USERNAME_FIELD`, `DAST_AUTH_PASSWORD_FIELD`, `DAST_AUTH_FIRST_SUBMIT_FIELD`, and `DAST_AUTH_SUBMIT_FIELD` are correct. ### Failed to authenticate user @@ -141,14 +141,14 @@ Suggested actions: - Generate the [authentication report](#configure-the-authentication-report) and verify the `Request` for the `Login submit` is correct. - It's possible that the authentication report `Login submit` request and response are empty. This occurs when there is no request that would result in a full page reload, such as a request made when submitting a HTML form. This occurs when using websockets or AJAX to submit the login form. -- If the page displayed following user authentication genuinely has elements matching the login form selectors, configure `DAST_AUTH_VERIFICATION_URL` - or `DAST_AUTH_VERIFICATION_SELECTOR` to use an alternate method of verifying the login attempt. -- Some applications display a "Loading..." element on a page before hiding the login form. This can confuse the analyzer. Use `DAST_BROWSER_PAGE_LOADING_SELECTOR` or - `DAST_BROWSER_PAGE_READY_SELECTOR` [variable](browser/configuration/variables.md) to instruct the analyzer that the page has finished loading. +- If the page displayed following user authentication genuinely has elements matching the login form selectors, configure `DAST_AUTH_SUCCESS_IF_AT_URL` + or `DAST_AUTH_SUCCESS_IF_ELEMENT_FOUND` to use an alternate method of verifying the login attempt. +- Some applications display a "Loading..." element on a page before hiding the login form. This can confuse the analyzer. Use `DAST_PAGE_IS_LOADING_ELEMENT` or + `DAST_PAGE_IS_READY_ELEMENT` [variable](browser/configuration/variables.md) to instruct the analyzer that the page has finished loading. ### Requirement unsatisfied, selector returned no results -DAST cannot find an element matching the selector provided in `DAST_AUTH_VERIFICATION_SELECTOR` on the page displayed following user login. +DAST cannot find an element matching the selector provided in `DAST_AUTH_SUCCESS_IF_ELEMENT_FOUND` on the page displayed following user login. ```plaintext 2022-12-07T06:39:33.239 INF AUTH requirement is unsatisfied, searching DOM using selector returned no results want="has element css:[name=welcome]" @@ -157,11 +157,11 @@ DAST cannot find an element matching the selector provided in `DAST_AUTH_VERIFIC Suggested actions: - Generate the [authentication report](#configure-the-authentication-report) and look at the screenshot from the `Login submit` to verify that the expected page is displayed. -- Ensure the `DAST_AUTH_VERIFICATION_SELECTOR` [selector](authentication.md#finding-an-elements-selector) is correct. +- Ensure the `DAST_AUTH_SUCCESS_IF_ELEMENT_FOUND` [selector](authentication.md#finding-an-elements-selector) is correct. ### Requirement unsatisfied, browser not at URL -DAST detected that the page displayed following user login has a URL different to what was expected according to `DAST_AUTH_VERIFICATION_URL`. +DAST detected that the page displayed following user login has a URL different to what was expected according to `DAST_AUTH_SUCCESS_IF_AT_URL`. ```plaintext 2022-12-07T11:28:00.241 INF AUTH requirement is unsatisfied, browser is not at URL browser_url="https://example.com/home" want="is at url https://example.com/user/dashboard" @@ -170,7 +170,7 @@ DAST detected that the page displayed following user login has a URL different t Suggested actions: - Generate the [authentication report](#configure-the-authentication-report) and look at the screenshot from the `Login submit` to verify that the expected page is displayed. -- Ensure the `DAST_AUTH_VERIFICATION_URL` is correct. +- Ensure the `DAST_AUTH_SUCCESS_IF_AT_URL` is correct. ### Requirement unsatisfied, HTTP login request status code @@ -199,4 +199,4 @@ Suggestion actions: - Generate the [authentication report](#configure-the-authentication-report) and look at the screenshot from the `Login submit` to verify that the login worked as expected. - Using the browser's developer tools, investigate the cookies and local/session storage objects created while logging in. Ensure there is an authentication token created with sufficiently random value. -- If using cookies to store authentication tokens, set the names of the authentication token cookies using `DAST_AUTH_COOKIES`. +- If using cookies to store authentication tokens, set the names of the authentication token cookies using `DAST_AUTH_COOKIE_NAMES`. diff --git a/doc/user/application_security/dast/browser/configuration/customize_settings.md b/doc/user/application_security/dast/browser/configuration/customize_settings.md index 733c4edf43e..d96f73e7c13 100644 --- a/doc/user/application_security/dast/browser/configuration/customize_settings.md +++ b/doc/user/application_security/dast/browser/configuration/customize_settings.md @@ -100,7 +100,7 @@ This can come at a cost of increased scan time. You can manage the trade-off between coverage and scan time with the following measures: -- Vertically scale the runner and use a higher number of browsers with the [variable](variables.md) `DAST_CRAWL_WORKER_COUNT`. The default is `3`. +- Vertically scale the runner and use a higher number of browsers with the [variable](variables.md) `DAST_CRAWL_WORKER_COUNT`. The default is dynamically set to the number of usable logical CPUs. - Limit the number of actions executed by the browser with the [variable](variables.md) `DAST_CRAWL_MAX_ACTIONS`. The default is `10,000`. - Limit the page depth that the browser-based crawler checks coverage on with the [variable](variables.md) `DAST_CRAWL_MAX_DEPTH`. The crawler uses a breadth-first search strategy, so pages with smaller depth are crawled first. The default is `10`. - Limit the time taken to crawl the target application with the [variable](variables.md) `DAST_CRAWL_TIMEOUT`. The default is `24h`. Scans continue with passive and active checks when the crawler times out. diff --git a/doc/user/application_security/dast/browser/configuration/variables.md b/doc/user/application_security/dast/browser/configuration/variables.md index fc5ddcb4ab3..41253cac9ea 100644 --- a/doc/user/application_security/dast/browser/configuration/variables.md +++ b/doc/user/application_security/dast/browser/configuration/variables.md @@ -26,7 +26,7 @@ For authentication CI/CD variables, see [Authentication](authentication.md). | `DAST_AUTH_SUBMIT_FIELD` | [selector](authentication.md#finding-an-elements-selector) | `css:input[type=submit]` | A selector describing the element clicked on to submit the login form for a single-page login form, or the password form for a multi-page login form. | | `DAST_AUTH_SUCCESS_IF_AT_URL` | URL | `https://www.site.com/welcome` | A URL that is compared to the URL in the browser to determine if authentication has succeeded after the login form is submitted. | | `DAST_AUTH_SUCCESS_IF_ELEMENT_FOUND` | [selector](authentication.md#finding-an-elements-selector) | `css:.user-avatar` | A selector describing an element whose presence is used to determine if authentication has succeeded after the login form is submitted. | -| `DAST_AUTH_SUCCESS_IF_NO_LOGIN_FORM` | boolean | `true` | Verifies successful authentication by checking for the absence of a login form after the login form has been submitted. | +| `DAST_AUTH_SUCCESS_IF_NO_LOGIN_FORM` | boolean | `true` | Verifies successful authentication by checking for the absence of a login form after the login form has been submitted. This success check is enabled by default. | | `DAST_AUTH_TYPE` | string | `basic-digest` | The authentication type to use. | | `DAST_AUTH_URL` | URL | `https://site.com/login` | The URL of the page containing the login form on the target website. `DAST_AUTH_USERNAME` and `DAST_AUTH_PASSWORD` are submitted with the login form to create an authenticated scan. | | `DAST_AUTH_USERNAME_FIELD` | [selector](authentication.md#finding-an-elements-selector) | `name:username` | A selector describing the element used to enter the username on the login form. | diff --git a/doc/user/application_security/dast/browser/troubleshooting.md b/doc/user/application_security/dast/browser/troubleshooting.md index c770288301d..3a9b36a7484 100644 --- a/doc/user/application_security/dast/browser/troubleshooting.md +++ b/doc/user/application_security/dast/browser/troubleshooting.md @@ -153,6 +153,7 @@ The modules that can be configured for logging are as follows: | `STAT` | Used for general statistics while running the scan. | | `VLDFN` | Used for loading and parsing vulnerability definitions. | | `WEBGW` | Used to log messages sent to the target application when running active checks. | +| `SCOPE` | Used to log messages related to [scope management](configuration/customize_settings.md#managing-scope). | ### Example - log crawled paths diff --git a/doc/user/application_security/dast/browser_based_4_to_5_migration_guide.md b/doc/user/application_security/dast/browser_based_4_to_5_migration_guide.md index aba3406ce9c..f64224ab217 100644 --- a/doc/user/application_security/dast/browser_based_4_to_5_migration_guide.md +++ b/doc/user/application_security/dast/browser_based_4_to_5_migration_guide.md @@ -79,6 +79,7 @@ See [configuration](browser/configuration/index.md) for more information on conf | DAST version 4 CI/CD variable | Required action | Notes | |:--------------------------------------------|:-------------------|:----------------------------------------------| | `DAST_ADVERTISE_SCAN` | Rename | To `DAST_REQUEST_ADVERTISE_SCAN` | +| `DAST_AFTER_LOGIN_ACTIONS` | Rename | To `DAST_AUTH_AFTER_LOGIN_ACTIONS` | | `DAST_AUTH_COOKIES` | Rename | To `DAST_AUTH_COOKIE_NAMES` | | `DAST_AUTH_DISABLE_CLEAR_FIELDS` | Rename | To `DAST_AUTH_CLEAR_INPUT_FIELDS` | | `DAST_AUTH_REPORT` | No action required | | diff --git a/doc/user/img/markdown_math_v17_2.png b/doc/user/img/markdown_math_v17_2.png new file mode 100644 index 0000000000000000000000000000000000000000..d2faa474dd71344da79287dd9234e2b4ffea441d GIT binary patch literal 16634 zcmbulWmKF^&?bBj8VC}Cy9PoC?he5T?hb)q14D4v7!q89OM<)31lQm$gA5v6f;$BG zhUeY0Klhy7k3Vqk?&_|pzPhUGqQW)Q6mYRAumJ$TRZ^7I0sssI0HEMJK}CL1i4Lnq z{%Y4y(UE(2c(}N@xVyW%zP^qKkCKvkh{D2>k&)5Gg(YKS^QkF#M|-E1 zrry@pR%mEMQ&US>X?ap&^5Nl8T^;Px$FJ|LY}M7YXQpR|helFT(gXa1ySsaLclYk^ z?|r=e$Hynq(=u(W?GqBd$HgUr?IFQIpIn^X$w;Y%1jRTx_!t>pN{GvZkaQ#gfaJN7 zthA0d=X#2s%V2AI(6DT-d0(#BQ53#Kzv3%a-e5LZ5ci92+jAmERNf`!N@VQnV|1NtONd4X z86+R0(YVKQ=rDP#X#OWFlIy0#{&GkEl`Km6m3qFE7CD8;(3~B^4oL}551E^6c}8wf z&T_vU%P(FysZBC2wdUOn`wm|v{S-5cUm07CaHMr4_hx8BKK|GEfqYR2g_Kh7BwLaZ zf+2Vej;Ds+qx@7kKZ{9RT3ca_q|c@x^DKWF+BDa75n6>-6~8*>S5aEudt?L6A0-WQ%e6B7V0j?{+7yYq=LLB z41n3X?^}ec##D#z*gqb~U2Iye>G#har1rO{;?#qmtA0=qysy&*(0sR|$wWlH+Mcuq z_9Hz*O+)-PU-kaWy))s|BXQgon-#ICcA3`Q3}Kto0j92g$64-q}Bm;LP0$FID8C@+5K* z^~xVt63?$Csyo3^(pNuLeX+bNYhJ^+L$uv#zd}O}u&a=ja!{kGio4-ON_I0BC zJj^5l$~~dQ!c-!q$v^*{{whs3dyqjecfE_HQOH=@vvcQhu1lZac27ED^D)=DRx)mh zn+t$%s`*077kFZ)7{&q@LYJ(~;}JO2_9W9D&#WS=>u?QNh10K7HVum-p&+vo*;gYB zZ}VJ#^!@BXG~}^NJL%c)b?3hA7Eu3oz~+DoAq#2pJvO3=>pYt@HFg>_Xe_F74Jz*Q z7B<>uY-^0e$B&!K74S6{T`Pvdcm*}`EjgNuhdqC zMC3Dfj{$<=l7?%!FO(ML(x?6O9Ir+LE5!H742^RO_76-VwHA+Dn#ypDnJvvP40G5k z{II-z^uB0-29_Vib?$TeadkCZ8AzT773>X6 z8SQPaOD;n-pXTYIN9@$8($B~81m(c^A!65BnA>?1-zh{tR5=b!`Gzzz7Nh*g#U(W~ zP)t?!4?7D(Z+kmWcoZL(rlFT$V3=+*xCi-q)9&-Bv_j{j@+X6$E;6gg{PL2%r((@p zR;}RR^|l4*Ii7deAJCT-e$*bB;>1L-c3)5=TF+9<4Pt zj|1r}pB`h&<-aiB=qOP!zEA#uWt2VPnXnG9B*8%I)ZMWB@8*>nHB*_fX1*d*?3%K+N$b_?(4||E(#_hOgm2+6xrn7TlnLBi!F*l{HSL5vn+4 z2VWG{;l`W%DV!hy=1!*_Z4m_#LK<)L+4);TLyrf5O=T=aiL2*X&~HJP8ATfYYIXU) zflfmzPXr=cJ0y*PG$9gswWzKeVBMj0!kn_$f)L2Z(ir=}ZfI)OOc?Z+ohraiZ-r_asu<$cT5ZgjnZeD{F3) zR5YHq+7S6?jo>4%cI6irf;hf<8a8B|SMyeHuw59NK)*=8*nEdI0!< zcu~K=eC2=r7{DK1LkJ$wYL*KjpqW;NQ}aUhRh;0Ju{u*}K}-d$RU8J^|@4`G3@h z%{_a}Tn|bJunWrF>#OAg2xx;7d0?eNSMo&4mw7il6g+03zd=a%Sjem=MHt<$dg}07Tqkb}#e& zAGzg%@~1=}eigFY4;!EU2*l6g+#v!hGmXv)vjDp;xdD?Gq&-?I9a5M(p~vNm-CQj* zjAJv^|J@T~x{L4E5)g`K#ubwV!tE#Xb;55gFFP*n(>FjoqLT$8+?9+<=tmh*u|z97 zF!}CJTd@|0bslW*zr{S`aL?iP-paI@TB(-sK{rBXh10IOOjb~#k$soB(l?LN#0Gk; z;t{6&CGjsTavW2=im$c;mXB*>zt5G+*g@agN8UR z*s12LXFd^7m9W#Mym=e^u7DT?bw5+UbPA|kfOjzaFhGqJviaW(h%h4$QSI_w+<45n zTfZRhZ6=Hr8B@m~MTr>2?ojOKJ7-gZ7I?r=t1{ ztW0`he|f__GfF2V$=jf1r!4g)aF}TOp%js&TE~C0tcZG(JavJsvg1i*q4%K_2ARJs z)Emk@)SnP+Q-L-{Wg7bbtJF(>^FzZuMA!2N_v;%@4N7}~@dtb8c&+rW-y_R6EwDed z625vv3zwN^!nW|*9?<Y%kKR8DCt2XrCa36lvBrQC36g!$PknaN6wqm2?k1OfPjllr zqLq3PU?#H$!DstWm95JP*Jjg^E;RdRH<#FH2B!sk(QpU+dLzGeb|e%~Kqg!A39YRa?uskrRnKI5|t#N+qz@EXiE^TksUwk^O&)JllM_=XJ z_%f-(VLlspjiY3Sccy^$^g1)wS|u#N1O29=G+d9@zCo7cJXB_~Ao^DA`9v7m9>&5O z2;5@)vev}Nuwkyz+qp2U`ROaO64ODC^gNB1_r&=Em><)=mkrJPJ)z)C1cYE$PQz?R z4+f6W0x5U|(0HFas)q+VuJZ(oGL)kfRKKVpaJ`t)!so2!U29FkP)GM!jVGR@2MUfm_=CF@=FPMY!$_-Rb!yPgh_K;pw|bWZH#UJJ?9u_|1P{s zG%eWYzzda*v|y2J-A7_4EXfC9nwnk&nkO(Gc<8O5mE34@=iO**%Z=luXsIyX6JA` zR>o5mG`I2gvR(R{7YG+yEPtVbFaYu!LxbhZn;-QRFp@7@h1Mpim+$VlqSYA&Qhb+S z_Vm865lnYW$WSh$c16d*UtJ1R5p9r|TGP9u;-bj|=a)IDI`e++RK9=cZAw%}{0TFc zB6+w*^wR54;t_cpuE6B;VeELsI>Dk8WO@w$9(o)Qx&bG5D%@X@>?rTBAzTQW%_u&} z-N>@e{&>qXHQgt-45$1p8uX?1?)tPl|gm6A}w2`mY8<%VMknxM6GnU;^D6*{f2Qb z-M@;)lcAo^Dsz}OZ_s5(Bf+-ne_I#6x(KS|#IbPovWmEoAq%kO%#6Tt7uv_|Cd=Ic zgQy6>ei%ocWg$+FMsNndFy91jRsP2cvkDB0g!b?7|A6Jj?%*Qvvm>a^a)q?AWdFmS z%KRnol#NNjt}}TWDCM_fh-=-`_f4biPjiup%ly_Xouyb+bXDbA^N9?Yy||nxAg5+h zzHj^H5`i}}C9j$>erXkDX>;sTfJv1<8afD0SnDxB&#)JiTdZUZr*ABMUb|wYF(SNe zA)F#>?T;dxtTgz^@3zbgIX9 z1j85cUQWIUtW51NJjJHqRE53Ad@89;X@7@-MtjX!1j&=(;%+MzKn&4)&rW>7!32z4yJNlI&8nh|-eu0jwg@4PBMeF*#sYsO0{ zjW5^S5CvPoCVNqWSECs+uE^@pc2mJ`dy~eRM+lz%nUQkvmL^>EA*H7qJ@L!HbK_Gh zEk|9M+9!YC2d)L-a5}b?!BDs;^H_dUVM0NPdl9Rv@Z~ENeX9PI;$kgdo{=Tz$4Z>CPi%n}8EYu-f`GYSkSQ(7Y zGsWEEP#-tN9JM_ScDT785IOA~og!gB=eurx!O2?5zW>p*4s7h&&Ui0?e32BZ?wvOg z=^p*F1%LCKI{St&GS^JsFRRZW4vBXsI~DR@E1_eUaA7?8v`HrdBJLie;d7cX+-ztP zxYP&jzA=`2o-qOD&F#DUhIkfbD?^p%FbMX0AcgW@{{fm_1o&^#`^ME_=uEK17Q-Yo zxV#IPph}F5P(*aAWZlTWa~bI3vE>4)Y1U6Fb-NThYH#HOl=9=Tsh_OQpcduj|K26y zj5^&@zkyq5f_A3G*zH%B2&(d6MWxbOcI-Izsc;vAKWve^&qd zab}%p{V?~iJ8CP2B!C==u_<{D4exm&Kk9A&3V~rX@S1f{bTENrv}oa3F`@ui*g30c zUkd9Z1xi;a_=QvBA*gakV{ElJ&}vVYUCySbJAy-3_nw75K&h%xh$Hm4 zQhP_lVz(NRN3duHpk8%KXVQ<025alT-sD?C zpWGps&~fqzJh3>E{U4KTk_lV~0xAStpSeCEt|b+e&`tN0>iemT6^ZdiY(URg#AyhF zVA#}s(=w~`z^sGM5!;))gK84Yd%zh}h4`vz)Fl3fqJP0_^`l7vqUhVR02QUI_OlSA zyKCSi0g6*+6v$LVuxJ38ui?CU+g3DuoSIx&RPKqUJ!np{$141z{EQ>zg`jje1cE9w zj{fN7>AWX7iXREuWL`Y-y$EE-voVk zwS}hU3c)*@ik3}1sn>KTmaoJ7BI4GVtL9SopL6FjE=Q}*5r-^pRo#7xjjCO5q$pGs z9$9C{oDy0`S$O#Xj`;RU>wwDlx2ehSLTXhtjA_yy)!P8k?gvI+4P4)NPF8!~k!i^E z7z8=t7YcbWDc7Wxi>gh`*UdGOzushYn+Hm9UM;&82?)N06gHEk&5bLL3HmslsyC9k z2(~=^e!{$c^fsA8C8iZ=o`&7U#Nc?c4@VL*x4{=?IH+)DIeFwO*C9dA1Q{e><3isz zC2MhJ|Fg$07YyXyels$!!{7LIdj2ShKpwty%m|-YQFb0)*-V`^+X0uE@L#SGj{ylO ze+dkKb#f#*?uo{#EV1F9hU-L`*4c^ihMRKT;9neo6h3lAbZ+04PE__r^uWsR4jrgs z;6-i%#~boqyh~h0n84lTj7i1jTEEJshX4CUD-^)~Ht|{yHnQ4r?6jFw2U0X1WgATO z;dXUSOWn7!M~#Iod5Ohxe${(}04N-+{F@^JtKCJD8vI3!yL$x+Mc?xRD^E*aNffGh zDTcQJz`1+q)G-T?Lk0oPQpb1j$fDX%0G4hW)SS^NYquL($aj(dtIG3#PY!nC5m;)q zU5-wBF1YqGl2*~D1{;h1XRuZrZW*vwv%6ce{P2lbq$i1WhwsPt)RAjbpN)(rTb>}Z zc(t;YDHhh(%;%p*xp{dho&5vj+rM16C^(#;sGrL`erp0x=ea176*UqNDzR_Vct@wE z8q$iy7sM7flH=q*A;HYDvau2Psx$G#NoSGweLRVVAxOs6x)$C@O-)^ts0z~=+5U(9 zyu}UFf;v2BV`FpC=aPp4l|nzr$u;l6nxW+?6hrPngU?T_l&wQIccc7`BvQgc%?CM^ zO(cYG-@b5Nr>Wbx5N1JRrg+?Z4{}p}wB)^BvQt8qC$>7&82|akZsvPB`-!Vy;rx`; z)NZBOeNT0pg=Cd^>IK%BRpVcN7mNW{^}^jq4S(M1CO)K|?7r=qvdyGJaWneX#Vehy~Lad`=j5c-p zkWwu5tmdtD8=8;f@X%06y?+q+TcQ>}#FTs?jr8d~1g`)dWt$cjhcyOU} zK8Q0Z=$|A@za*ju(Tg*3UbvDUOy1PBtqT$%yPu1Z{eR&goukk4O^8-o@4 zD8EJ1*<1U0nLeZDNr~i~nth!UUBgTkbL@ruB+o}zDC1f$tC`!<8nt94df^{yZB1kSM(G=}Q9f_SUg*TFqC@3$LoUOEC0N#)R z7W*DHl_>w-`eWf+HT;%a!ua2J_5b+~sf`0kQ8pVO_%AOxA<(|7%|-(R*GBnL0y@h@ zSF^%|2eL_p_@rb(^}zE>eq|NeD9yyzJ*|V64|*T$2G9^(YyUnhr4D5T9w1=uNxEBS zy!)NB!ScD|c4t5K1w~-?d=ydgynFXgcyBE2_$Y>}94FK*1O&EYY(%%mS+vMPA6jA_ z9MYQ%1vZ@0I+*|3{#Y-@K|eL6>=j|1Y{_IJH~5; z)X?urZHyiUw#ECPl!>k5C!_hb{fCTPS_M$VJsKZ{DrUVttdU49Iq)N*?=1Y@4mv*c zWAr!Y*!B7)0ojInb}p>Sl-Sbp*3<^^{AYdZB>48H9iI_oNL_>SRH1Z#&n2+!6b`%A zS8P_q3w=R<%`yhQHF}&;(PAzJ!6sT8*|NGi$K2zM>-dy@}2^?{7YgkNLXU(O1}n$*!Q!!52jVNc4v2}^N- z@V?a14Yu^b8lgMW;tLZAT}sYBff)_L0fw(RPMAEyD^#rKF8we?iu0{-C9 z?C?JlCr|sQXpvqhO%l;fe@K2NM}JRZ$rQ7TKN_px->Gc*UOGXYufIY4q=DUxzGXizP$*)AiW$OUfC4SFqimE4*=Bl}Uf* z(Z=wV^o>~ll+6-Hw(5i(Uz&M=K>q|v)!TiaXiN)3kcIFWo(9_Pu}Lo~Gf8mx!dG?x z9-kz%8~D-Na)Q}cN~IdFeK(*g;c#sZ9Vkl_iX>gpMZY<)hLWP=d9nrNS*M zKy)-ey9U`ogMNPPuR&GHa)85ymYBUb;UbEZHMxR7k!Wh4zJN=Z$x|uQ`_kA_*yK0`FQGawT{g2;;fLGsucR)4EEeMoIZ()ib;wYkBt-qD+Mx_A1+bJM+ z0pnXhCFa$KC}SExl$#6^^ZOQ-#8HI7gHzdE>uRM*T5tioT;Aft9bwu(L^8Qm(}9XK~)TwSQ%6J92;VsuV_C`c!R~?O^TPRi2wLC#OVJ;u!m7 zW=VNh+caXNah+S@@fSkrWlo&lJTL9hHePT-rNE2R;p4MSoe-B+#1vi?V#*UecqU1m zZL@fI;J?d|t*5MOY#g>{08wH3z5a>wfs}+yIB(pJuT29IPKA0~M6@98>;!vhdNJ zL~kMuAlx3;<{X)7 zqau_Ll)k+UX-k|T0m=|wpm~#G8>v^nKUUcR6R2uXKOF(mOGi|QtR(y%)lAn4vPhIH z*HP9?U)TH1XF?M#H}p?UH(fsW`uX-URE`{)>i1ixvXTfIRRd;2{q?VvnoJjB9WNG` zCjh4FK|~GzHLa&Ow9Oa)@Rz)S2fVQm&(_?tJP2?D9< znV3rQ5ShH{Q+UC9Cwe(Ha|jsIoBG^vx+&Fds^EV`Gp#9R`+tmP*IW17ur}GDjv4Tz z9iO=w)@JK=pRC6vaO>!*vg+>A zD?*ZdcCGaYrmm3@6#TYw*hh5lOY3E(+t4Xytil;NAfKHY3rK&i*KS>bn5riqDr&)e zf5ksW)%tE=gF(4!eZUWxnfxJbr3--op=zV_ETsap?MG2Zj!@!|(mw>3xgW~_;~lA1 zM9Bvf?BhoTlHBl>s6Ic_H!Wb;tn6yld-VM-*ec^`&Q$BBC%ip>Hx)naIvIq+YF#ys>^>i^y`Q^r@Vlg$-x?KQ}k;@QX9g^Kc0Xuo*rE8 zPNx7#RmT;~{viO>YWSrPs2mL`gZ+^Y`~Kf%t6*NgJ3MNyqfW1>??&TwOHdD}l40r_ zEr>Y%h3mF5)I!Du-_ z(HZuK7Q4hm#!vE8rtnLNE_#==Zipqo1I5j4TvRHRAU5>wtd^_ z+~S-6eMKz%qRbIP!9N>8ui{Ned{;`{r#oNv&?X5JvtOxAy^j#aC*Bf_Bs9dTb)3rK|IS%sq2H({0po%;! zjY8pyDND(u^4EuKGN$CeWz7&t347Et!1fbk6nl0~)&S1MBXdx(cZwUsbcCW_gGB?CroQbWUvlKYz@rCspmT59Ngn=w zen#+6wqNtoEcT8<3CK}LzFJS+;L|ybQsGSc{eJEQhxR@EmbWl zz(!x-V10uWsLJn|UuwXNoJ@iRJO!E?q{%z^ElE(9ytO|TX<+jo5J9bB8`DS(e4t)ENWv zY_#5bqV0P{`|(18d%f4H$vogSiLB}66jZy-|MDOS2m0iSpEA$wf$00XD);TZAIMx% z&=!Fsl5W^QWy9$eb&6y}Y}_Lz*%bcFxu~<^49f7{H8;6d#mdtB}yJZs-EY0(6!1AS=%b?&*o|Dkwr!&GU{b(nC zq1uhMF`VrQ(bt&c6H#=^wOG(|&nL-vJK{bMn8pTQOys{oXXI7HvnjyaMfV3rjhS8F zE-Y^SAe1ow>*b-L4A{iSrz^#t9{94q)Y4N^ColGz#8;rs{_>RN(klbV;TenVd-?EMoS>-Fs`SClEa(_=vom1zqBPHlVmeMSwuy0SSk-R|d z)4F_tWHgg*q>aiRmrXB--GdRFbhPX?D6}k4=1jNAxJ@j{lUHCsn}aboUMA^PsOg21*f_#oHOH$TygWh00vR$6_-+xryu5y!FZH77bfW$2wm;k@ z_S0+<+1l7lud-@n{nZuLvfI%XEBqBI-2KRqj6KQMd~2Lvh~NM{FaJ`zZK7qW$-Yr6xO_rUH2y36f5u)b<=WKx@NE3BKpgxrbKtK?ALfjio4ns# zNACWO)@k*8G!lR9qy5_-#7j7>VEew%HXSX0UIVmiw-e;9ti`o_yUpUtyGQi8)pal0 zPr11c6+I;Cbd(PhlQvZ3dC>sGkT!$te zG|Tn&5Ja{|uZl7GbR&@@hc}K#`H)4O4B|%0>+~Ep0e-fqyekDQw?ul^s6GGtn-vWo zk+BH&<0Mwlw;hSVc?cW1{-W-0@RO0 z0wHRxM{{%{n~6Vzahb=fo5T(cHeR7=yxrKP{-L3=jJGSwY?Vi4CzptW>0bzx6;q%1 zQ^<3O;tzTx5aWIKj3LA;@`BK+_Xx|Jn)}`m*Utp=RC^!jonJhTw>m9>za?o3K$1Li zc8^L-5)vR72(Hj@iL>X#mdF#wuvF?`H7V+9QZ7*Coj6xAHg8JlOuc9GUo-7SqHx%P zt{YRzG>H3UCXCcTQ3;K3fBkul)Z0bIpA`{E7Q!!h`}*xb8$AU#{$N{8re?XR^i2*m zF1^FKJo&;e-9)7Cw#dTLkpBkWFLFmrNud9fHy3rVDI3B=-X}fzHZL+|>=D0ePsQ{^ zocpdoY74GsXwpRcN|+WQIgC#_7aqcde;0VYZ}$59@g)1#Rk&}E89CsLXmD8L1UUzx z{Xf$i|9=I?=z7bZ4#$CjD0x#SG@wox9KskJ5NtCE0r&=tm{4b*gbB=HNF^J7Q<+B{ z``P*zj#~5E;u+eC**O9RA<8-oDpA$S5;ML_lJBOqf~+w^oV=jF1{{;`r`iRd6WDiu zx8Q1w0ban3GA12S%@JkE1~qtVF*~Swc_P@QF|rS`D{u>RQ|%DR0Wo3`^eO(Dvps3vi;GRzPwuTZ zwK)2L$f9z`x%rTXe)<;d2P!y=Sp$0SdrR2=PLriiiIb~~MkY5f6ZFFT2A9c88G_6%pCeacbOyX5mtk%*J+FFjhH+3Gc) zERVbt>~*7H(_Q=varebjns|Vh(R?_NtHSztRP2>@B+$6;;nPY(Df=EBbZDrN^OEB4 zK-A0tF*N+u-#vE0f)A(HQ6JWE!n`~%Fr+Pxv}(`X!}a(SLSYJvD@o6%Xo|sm+%UH} z+7G{HH>vQ*9ei{Jn&16RzBToX92O`O7W31>I-Rn~W(g2vD;jt;G`%6i#a~mJ)2ctP zh#{Shtm{JUpw}yWk!K@RdL8!ctmw|o#Q2oA&l!v=xD+ry9ta!A9MLVgL~t%@em+@f zxxxrhC&y3}jM(G~Wo7_}MT;|By~<*7gF zDX!BqpG*xzemK^F_P?rtEkyBhR^TZ9rYU0Mjs1)4>`CQNGoNx=(tW4(5y7Yn5sWM! z0e>T!5f9bw$;yq1!yy-KT(<6fmSsvJ45|F84~G#zJTDBb_f>y8C>x@?v6&taaOH9z zi;BiPR{LoPp{IoH98g|kwKF^7V0%pz4HTnsXcnr93P9KzlLmmTwS{(lK5lpBQ9_Og z@=1W&AZr#$dHpS+S}n6NG_Vk1-{NREA!qzWkq_a8khYWD!|JgNkpjK1i^tP;cMg0q zYrqD{C{p{eHrlnGL=hEn{2W4eWg%Ru4VQbL4jBD2@5Y}hfHjYSR%BDThV;`#P{R=! zL++XeAGgDI-cJ==DC!!w8%Syy3uSVQ{urI5(@Gc3(pxy3uxwlGv%cGz(v#7J)I7zT zF_0{OJ)aokh28(k4!%CtyI=N_>UBUFgTsK-y6f?h3F}f&{tV?TqGdcv$-gf zo(IRtb#n)6{1wUkG3!~f>j&AMmF*(Yc$m)=LsJg;+4*!#b7<*vO*4Rsqo5qe(Hmm5B zPvw3w#*{eE1^)^^vN9fwB6QyVVDF7qB9AQi$Ya6U1#Hi)eLgrQW76dcKov?5jgjKA z;BRxh0)N1_SPDG_af20~_2C#9UvH$ssTZVGwD$*=fxQ?(D= z6?5|&E96tr!SQMwRz6wS*S0z{>3hdy3Rz%|Ww(!tyGANR@7zx%%=P_kz zPlj7RqHcv*tR@Ub9co2`fegEebE#UKI$dH=U1S5#O}@kYgf(_RPO2gL+7GIaS@p&m ze2U0tdPQlud%;?n7P&TmkiSoj81`!BnM4F=2@vh_Z*T1=oR%lh8 zG^W}lRL==N%YzHZruja)I9+kP|53>r8fao_ilJuu1Y<~gk5g?J0OExg8I>6vZ8DSa&zih9Z7XkgosLip;BE)=S$fd)>lSS-Y|XKz>3 z3;7BAB>h<5qC&C0wq#-vAbWd$VhN2BkLKKXO))gSot%o7&*-$Zr zWRJj78KScC5ak%xAew7ZE5UHYSIQm{NgZR}*SrFHou%MU1YY<>W{sPm1X-T~1N0IE z*ovD!L2sp-MbA)W~(Ovz;Sq0g@h7#6=6f**Em*8-nh(mVTsU%Nl-h6M{gDTSt4PUtbL=wkjfk-CdmZ5(tp^ znp=w;S}d*wC*jfC1lN8+Fyqkc8?$Ss+cuG(bh28-ELclenZ9VQfKhUj>*lb0Eo#Tq z5vaowE6@)ar-0%Lqk$|CX9396s5eh&srx!)cdHN<*t}2q;W&WO=Z+BSVTo>(KLp3j z(aYieAT`{lpPr`A^PTdZ{fG@g5wa-YG-9F?I%5z(=Ej?b&dpbXpm6C9;2qJ8!l%~0 zzI|1Jx?rsqKa4+RV;%@>0L>bmxK+He37Y!0bovnW7Z$C8usaMMu(mIg3+KOgj%3B< zg9fJm4y=%Ex)MIpC)b(iPMv3hrM~)I^j7=_HH=nPNLvIPR|(#nUHv)vLU061Qn(s? z<*kG-UHz(!R4_}j{>=K(8cKAjotd8HRH;ihC^_C3StW*BMg!@L%bLD~?mUqcSLRaU zXF;ZJo)xtFwKoznNb)#MpUz}@M-QX->fJm@et)12WI6KiPpIQiSI1L zdZl(M^U8Zk`Ja!uJ(GKC?lzhm7SlF8{kmGvL_&VGg}3MC;LC_`hAI9v?O6{AW6!;} za!RfMpWiQQVDvaFx>4q*K2S)LJ!D^lT`MS7Ad&=!MfIKn5cc@B!Y@;S0d+QrXpBT|(8j3ul(uEr8`n=Ga z*AQ@Ml2Kt>ykKwp5nC#-OW#AKou=>vL&`KW#2?y^A1-pMCl3L~=s>?bC7IlGq3g9a2poOo@;W_0}g2iC;;MhEJjtKSSv>;q)wVk^3tAj23QQz{3T1R5`rA$vp~VW)mV zChKyyOC`ydAb#Th#iay>_SwL%6+(NY*!gFy`>pI z9w>>-LVxd|>JBhI_(8B%yB;L1yVAfZA+4X0QOd3k@%2UGg*&qeP-z5tjPvF(ddBZi zRy2Z^KwMpiCHtG*s=QCcE(>u`B?YIzw+?=nKQh96!PR>Fqh9S-DW`b&nkkPVd-*iy29lOY)nfp{y91Uic2*7*YjMGqHRy=i3`K z#Z6u>0W2_vZSMG^{>1ZP16eW-^-Sd`!M5XEcBm^XljI!Ig}}-jU*sHc)n7-U-}R7N zDqs5lR;jNXes-$Nyx03hGSN)NYte+Mf0+c?|w$4w_p zT&aNB)&PFrpz2wZRn5bEC1*xSF#(&Z zZ2YJ+o1=z;ZS}|r5ijAuR^)P@^yk33rU&Y*Rg1nxUY5n>9`h1Z0f(^b*DgpGbGl8I z`A%j}yx6yg!yeYOjNM9uxNe-QPyQQS4D_?LwY_CNv?+uA>Yt~WANn9F>UW4qo5hb* zV%+Dlg!kS(RzaUv&scKqLyJg|AtR}eyMR}o{?=twzYuUl>fuM;Aa-?fKd*WlbHi^8 z^+@nvP;~XufDh*0r41fC(fn17;-v;mSne`2lOi*wKc|*JBpL7ahdmLCMnik}{XLs< zqnUlc&|30)D+7vlg>n37R0(PS^jYl;i+V#bR+qHvWYLJSt2Qr6aIWV`iM($= zm|X+H*1s>qzx1Ffw{-C^I541G88}7GlpWNzFDc4T3D`j*D{Y%qIY^6OlS~Q;z4M!O zsdeTLt-HT?%I1Lfz<6_TUef#W0^zGG(;M2hrZkgkKkb-L<;mZ~WsD`@pK_r#!%qOU zp2^U|cs_7ii`Y(c>n)F&Ws}N8@il}+_$82{&eFeM-#qP5GJWSoz7Rvr$C5RQR+Rqf+Qo6x>YEs9)pgOT6QvPKh=VZmRd=mr|_Bxc0hxYt&vArLDz;E2b z9KOjWqsgX)?KXhg>k)y>z)?)*O%v$ C$$Iku literal 0 HcmV?d00001 diff --git a/doc/user/markdown.md b/doc/user/markdown.md index b77aca81383..942f9568a0b 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -1039,21 +1039,7 @@ a^2+b^2=c^2 $$ ```` -This math is inline: $`a^2+b^2=c^2`$. - -This math is on a separate line using a ```` ```math ```` block: - -```math -a^2+b^2=c^2 -``` - -This math is on a separate line using inline `$$`: $$a^2+b^2=c^2$$ - -This math is on a separate line using a `$$...$$` block: - -$$ -a^2+b^2=c^2 -$$ +![Example of math in GitLab](img/markdown_math_v17_2.png) ## Tables diff --git a/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_namespace_id.rb b/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_namespace_id.rb new file mode 100644 index 00000000000..1f66e024622 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_namespace_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillComplianceFrameworkSecurityPoliciesNamespaceId < BackfillDesiredShardingKeyJob + operation_name :backfill_compliance_framework_security_policies_namespace_id + feature_category :security_policy_management + end + end +end diff --git a/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_project_id.rb b/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_project_id.rb new file mode 100644 index 00000000000..f72e03c6d3f --- /dev/null +++ b/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_project_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillComplianceFrameworkSecurityPoliciesProjectId < BackfillDesiredShardingKeyJob + operation_name :backfill_compliance_framework_security_policies_project_id + feature_category :security_policy_management + end + end +end diff --git a/lib/gitlab/background_migration/deduplicate_lfs_objects_projects.rb b/lib/gitlab/background_migration/deduplicate_lfs_objects_projects.rb new file mode 100644 index 00000000000..e4191946ffd --- /dev/null +++ b/lib/gitlab/background_migration/deduplicate_lfs_objects_projects.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class DeduplicateLfsObjectsProjects < BatchedMigrationJob + operation_name :deduplicates_lfs_objects_projects + feature_category :source_code_management + + # Temporary class to link AR model to the `lfs_objects_projects` table + class LfsObjectsProject < ::ApplicationRecord + include EachBatch + + self.table_name = 'lfs_objects_projects' + end + + def perform + each_sub_batch do |relation| + data = duplicates_by_project_id_and_lfs_object_id(relation) + + next if data.empty? + + # After plucking the duplicates, build a VALUE list + id_list = Arel::Nodes::ValuesList.new(data).to_sql + + # Use the same GROUP BY query as in the MR to properly narrow down the duplicated records. + # In the previous query we didn't include the repository_type because it is not covered with an index. + subquery = LfsObjectsProject + .where("(project_id, lfs_object_id) IN (#{id_list})") # rubocop:disable GitlabSecurity/SqlInjection -- there is no user input given + .select('project_id, lfs_object_id, repository_type, MAX(id) AS max_id') + .group('project_id, lfs_object_id, repository_type') + .having('COUNT(*) > 1') + + join_query = <<~SQL.squish + INNER JOIN (#{subquery.to_sql}) AS duplicates + ON lfs_objects_projects.project_id = duplicates.project_id + AND lfs_objects_projects.lfs_object_id = duplicates.lfs_object_id + AND lfs_objects_projects.repository_type = duplicates.repository_type + SQL + + duplicated_lfs_objects_projects = LfsObjectsProject.joins(join_query).where.not( + 'lfs_objects_projects.id = duplicates.max_id' + ) + + LfsObjectsProject.where(id: duplicated_lfs_objects_projects.select(:id)).delete_all + end + end + + private + + def duplicates_by_project_id_and_lfs_object_id(relation) + # Select project_id and lfs_object_id pairs which have duplicates. + inner_query = LfsObjectsProject + .select('1') + .from('lfs_objects_projects lop') + .where('lop.project_id = lfs_objects_projects.project_id') + .where('lop.lfs_object_id = lfs_objects_projects.lfs_object_id') + .limit(2) + + count_query = LfsObjectsProject.select('COUNT(*) AS count').from("(#{inner_query.to_sql}) cnt") + + cte = Gitlab::SQL::CTE.new(:distinct_values, relation.select(:project_id, :lfs_object_id).distinct) + + # Limit count to determine if there is a duplicate, we don't need to load all duplicated rows + # (only 2 rows are enough for a project_id, lfs_object_id) pair + cte.apply_to(LfsObjectsProject.where({})) + .where("(#{count_query.to_sql}) = 2") # rubocop:disable GitlabSecurity/SqlInjection -- there is no user input given + .pluck(:project_id, :lfs_object_id) + end + end + end +end diff --git a/spec/features/projects/container_registry_spec.rb b/spec/features/projects/container_registry_spec.rb index 6ce2d042467..cd7191f6f01 100644 --- a/spec/features/projects/container_registry_spec.rb +++ b/spec/features/projects/container_registry_spec.rb @@ -24,185 +24,223 @@ RSpec.describe 'Container Registry', :js, feature_category: :container_registry stub_container_registry_config(enabled: true) stub_container_registry_info stub_container_registry_tags(repository: :any, tags: []) - allow(ContainerRegistry::GitlabApiClient).to receive(:supports_gitlab_api?).and_return(true) end - it 'has a page title set' do - visit_container_registry - - expect(page).to have_title _('Container Registry') - end - - it 'has link to next generation container registry docs' do - allow(ContainerRegistry::GitlabApiClient).to receive(:supports_gitlab_api?).and_return(false) - - visit_container_registry - - expect(page).to have_link('next-generation container registry', href: help_page_href) - end - - it 'does not have link to settings' do - visit_container_registry - - expect(page).not_to have_link _('Configure in settings') - end - - it 'has link to settings when user is maintainer' do - project.add_maintainer(user) - - visit_container_registry - - expect(page).to have_link _('Configure in settings') - end - - context 'when there are no image repositories' do - it 'list page has no container title' do - visit_container_registry - - expect(page).to have_content _('There are no container images stored for this project') - end - - it 'list page has cli commands' do - visit_container_registry - - expect(page).to have_content _('CLI Commands') - end - end - - context 'when there are image repositories' do + context 'with metadatabase enabled' do before do - stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest], with_manifest: true) - project.container_repositories << container_repository + allow(ContainerRegistry::GitlabApiClient).to receive(:supports_gitlab_api?).and_return(true) end - it 'list page has a list of images' do + it 'has a page title set' do visit_container_registry - expect(page).to have_content '1 Image repository' - expect(page).to have_content 'my/image' + expect(page).to have_title _('Container Registry') end - it 'user removes entire container repository' do + it 'has link to next generation container registry docs' do + allow(ContainerRegistry::GitlabApiClient).to receive(:supports_gitlab_api?).and_return(false) + visit_container_registry - expect_any_instance_of(ContainerRepository).to receive(:delete_scheduled!).and_call_original - - find('[title="Remove repository"]').click - expect(find('.modal .modal-title')).to have_content _('Delete image repository?') - find('.modal .modal-body input').set('my/image') - find('.modal .modal-footer .btn-danger').click + expect(page).to have_link('next-generation container registry', href: help_page_href) end - it 'navigates to repo details' do - visit_container_registry_details('my/image') + it 'does not have link to settings' do + visit_container_registry - expect(page).to have_content 'latest' + expect(page).not_to have_link _('Configure in settings') end - describe 'image repo details' do + it 'has link to settings when user is maintainer' do + project.add_maintainer(user) + + visit_container_registry + + expect(page).to have_link _('Configure in settings') + end + + context 'when there are no image repositories' do + it 'list page has no container title' do + visit_container_registry + + expect(page).to have_content _('There are no container images stored for this project') + end + + it 'list page has cli commands' do + visit_container_registry + + expect(page).to have_content _('CLI Commands') + end + end + + context 'when there are image repositories' do before do - stub_container_registry_tags(repository: %r{my/image}, tags: ('1'..'20').to_a, with_manifest: true) - visit_container_registry_details 'my/image' - click_sort_option('Name', true) + stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest], with_manifest: true) + project.container_repositories << container_repository end - it 'shows the details breadcrumb' do - expect(find_by_testid('breadcrumb-links')).to have_link 'my/image' - end + it 'list page has a list of images' do + visit_container_registry - it 'shows the image title' do + expect(page).to have_content '1 Image repository' expect(page).to have_content 'my/image' end - it 'shows the image tags' do - expect(page).to have_content '20 tags' - first_tag = first('[data-testid="name"]') - expect(first_tag).to have_content '1' - end + it 'user removes entire container repository' do + visit_container_registry - it 'user removes a specific tag from container repository' do - service = double('service') - expect(service).to receive(:execute).with(container_repository) { { status: :success } } - expect(Projects::ContainerRepository::DeleteTagsService).to receive(:new).with(container_repository.project, user, tags: ['1']) { service } + expect_any_instance_of(ContainerRepository).to receive(:delete_scheduled!).and_call_original - first('[data-testid="additional-actions"]').click - first('[data-testid="single-delete-button"]').click - expect(find('.modal .modal-title')).to have_content _('Remove tag') - stub_container_registry_tags(repository: %r{my/image}, tags: ('1'..'19').to_a, with_manifest: true) + find('[title="Remove repository"]').click + expect(find('.modal .modal-title')).to have_content _('Delete image repository?') + find('.modal .modal-body input').set('my/image') find('.modal .modal-footer .btn-danger').click - - expect(page).to have_content '19 tags' - expect(page).not_to have_content '20 tags' end - it('pagination navigate to the second page') do - visit_next_page + it 'navigates to repo details' do + visit_container_registry_details('my/image') - expect(page).to have_content '20' - end - end - - describe 'with a tag missing digest' do - before do - stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest stable]) - stub_next_container_registry_tags_call(:digest, nil) - visit_container_registry_details 'my/image' + expect(page).to have_content 'latest' end - it 'renders the tags list correctly' do - expect(page).to have_content('latest') - expect(page).to have_content('stable') - expect(page).to have_content('Digest: Not applicable.') - end - end - - [ContainerRegistry::Path::InvalidRegistryPathError, Faraday::Error].each do |error_class| - context "when there is a #{error_class}" do + describe 'image repo details' do before do - expect(::ContainerRegistry::Client).to receive(:registry_info).and_raise(error_class, nil, nil) + stub_container_registry_tags(repository: %r{my/image}, tags: ('1'..'25').to_a, with_manifest: true) + visit_container_registry_details 'my/image' + click_sort_option('Name', true) end - it_behaves_like 'handling feature network errors with the container registry' + it 'shows the details breadcrumb' do + expect(find_by_testid('breadcrumb-links')).to have_link 'my/image' + end + + it 'shows the image title' do + expect(page).to have_content 'my/image' + end + + it 'shows the image tags' do + expect(page).to have_content '25 tags' + first_tag = first('[data-testid="name"]') + expect(first_tag).to have_content '1' + end + + it 'user removes a specific tag from container repository' do + service = double('service') + expect(service).to receive(:execute).with(container_repository) { { status: :success } } + expect(Projects::ContainerRepository::DeleteTagsService).to receive(:new).with(container_repository.project, user, tags: ['1']) { service } + + first('[data-testid="additional-actions"]').click + first('[data-testid="single-delete-button"]').click + expect(find('.modal .modal-title')).to have_content _('Remove tag') + stub_container_registry_tags(repository: %r{my/image}, tags: ('1'..'19').to_a, with_manifest: true) + find('.modal .modal-footer .btn-danger').click + + expect(page).to have_content '19 tags' + expect(page).not_to have_content '20 tags' + end + + it('pagination navigate to the second page') do + visit_next_page + + expect(page).to have_content '20' + end + end + + describe 'with a tag missing digest' do + before do + stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest stable]) + stub_next_container_registry_tags_call(:digest, nil) + visit_container_registry_details 'my/image' + end + + it 'renders the tags list correctly', :aggregate_failures do + expect(page).to have_content('latest') + expect(page).to have_content('stable') + expect(page).to have_content('Digest: Not applicable.') + end + end + + [ContainerRegistry::Path::InvalidRegistryPathError, Faraday::Error].each do |error_class| + context "when there is a #{error_class}" do + before do + expect(::ContainerRegistry::Client).to receive(:registry_info).and_raise(error_class, nil, nil) + end + + it_behaves_like 'handling feature network errors with the container registry' + end + end + end + + describe 'image repo details when image has no name' do + before do + stub_container_registry_tags(tags: %w[latest], with_manifest: true) + project.container_repositories << nameless_container_repository + visit_container_registry + end + + it 'renders correctly' do + find('a[data-testid="details-link"]').click + + expect(page).to have_content 'latest' + end + end + + context 'when there are more than 20 images' do + before do + project.container_repositories << container_repository + create_list(:container_repository, 22, project: project) + + visit_container_registry + end + + it 'shows pagination' do + expect(page).to have_css '.gl-keyset-pagination' + end + + it 'pagination goes to second page' do + visit_next_page + expect(page).to have_content 'my/image' + end + + it 'pagination is preserved after navigating back from details' do + visit_next_page + click_link 'my/image' + page.go_back + expect(page).to have_content 'my/image' end end end - describe 'image repo details when image has no name' do + describe 'with metadatabase disabled' do before do - stub_container_registry_tags(tags: %w[latest], with_manifest: true) - project.container_repositories << nameless_container_repository - visit_container_registry + allow(ContainerRegistry::GitlabApiClient).to receive(:supports_gitlab_api?).and_return(false) end - it 'renders correctly' do - find('a[data-testid="details-link"]').click + context 'when there are more than 10 images' do + before do + project.container_repositories << container_repository + create_list(:container_repository, 12, project: project) - expect(page).to have_content 'latest' - end - end + visit_container_registry + end - context 'when there are more than 10 images' do - before do - project.container_repositories << container_repository - create_list(:container_repository, 12, project: project) + it 'shows pagination' do + expect(page).to have_css '.gl-keyset-pagination' + end - visit_container_registry - end + it 'pagination goes to second page' do + visit_next_page - it 'shows pagination' do - expect(page).to have_css '.gl-keyset-pagination' - end + expect(page).to have_content 'my/image' + end - it 'pagination goes to second page' do - visit_next_page - expect(page).to have_content 'my/image' - end + it 'pagination is preserved after navigating back from details' do + visit_next_page + click_link 'my/image' + page.go_back - it 'pagination is preserved after navigating back from details' do - visit_next_page - click_link 'my/image' - page.go_back - expect(page).to have_content 'my/image' + expect(page).to have_content 'my/image' + end end end diff --git a/spec/frontend/packages_and_registries/container_registry/explorer/components/details_page/tags_list_spec.js b/spec/frontend/packages_and_registries/container_registry/explorer/components/details_page/tags_list_spec.js index 18962be6f05..60554e44215 100644 --- a/spec/frontend/packages_and_registries/container_registry/explorer/components/details_page/tags_list_spec.js +++ b/spec/frontend/packages_and_registries/container_registry/explorer/components/details_page/tags_list_spec.js @@ -16,6 +16,7 @@ import deleteContainerRepositoryTagsMutation from '~/packages_and_registries/con import { GRAPHQL_PAGE_SIZE, + GRAPHQL_PAGE_SIZE_METADATA_ENABLED, NO_TAGS_TITLE, NO_TAGS_MESSAGE, NO_TAGS_MATCHING_FILTERS_TITLE, @@ -230,6 +231,65 @@ describe('Tags List', () => { ], }); }); + + it('increases page size when paginating next', async () => { + findPersistedPagination().vm.$emit('next'); + + await waitForPromises(); + + expect(resolver).toHaveBeenCalledWith({ + ...queryData, + first: GRAPHQL_PAGE_SIZE_METADATA_ENABLED, + after: tagsPageInfo.endCursor, + }); + }); + + it('increases page size when paginating prev', async () => { + findPersistedPagination().vm.$emit('prev'); + + await waitForPromises(); + + expect(resolver).toHaveBeenCalledWith({ + ...queryData, + first: null, + last: GRAPHQL_PAGE_SIZE_METADATA_ENABLED, + before: tagsPageInfo.startCursor, + }); + }); + + it('with before calls resolver with pagination params', async () => { + findPersistedSearch().vm.$emit('update', { + sort: 'NAME_ASC', + filters: [], + pageInfo: { before: tagsPageInfo.startCursor }, + }); + + await waitForPromises(); + + expect(resolver).toHaveBeenLastCalledWith({ + ...queryData, + first: null, + before: tagsPageInfo.startCursor, + last: GRAPHQL_PAGE_SIZE_METADATA_ENABLED, + }); + }); + + it('with after calls resolver with pagination params', async () => { + findPersistedSearch().vm.$emit('update', { + ...queryData, + sort: 'NAME_ASC', + filters: [], + pageInfo: { after: tagsPageInfo.endCursor }, + }); + + await waitForPromises(); + + expect(resolver).toHaveBeenLastCalledWith({ + ...queryData, + first: GRAPHQL_PAGE_SIZE_METADATA_ENABLED, + after: tagsPageInfo.endCursor, + }); + }); }); }); diff --git a/spec/frontend/packages_and_registries/container_registry/explorer/pages/list_spec.js b/spec/frontend/packages_and_registries/container_registry/explorer/pages/list_spec.js index f6788c9e87c..dc5ff35bc6c 100644 --- a/spec/frontend/packages_and_registries/container_registry/explorer/pages/list_spec.js +++ b/spec/frontend/packages_and_registries/container_registry/explorer/pages/list_spec.js @@ -16,6 +16,7 @@ import { DELETE_IMAGE_SUCCESS_MESSAGE, DELETE_IMAGE_ERROR_MESSAGE, GRAPHQL_PAGE_SIZE, + GRAPHQL_PAGE_SIZE_METADATA_ENABLED, SORT_FIELDS, SETTINGS_TEXT, } from '~/packages_and_registries/container_registry/explorer/constants'; @@ -646,6 +647,63 @@ describe('List Page', () => { }), ); }); + + describe('with metadata database enabled', () => { + it.each` + event | expected + ${'prev'} | ${{ before: pageInfo.startCursor, first: null, last: GRAPHQL_PAGE_SIZE_METADATA_ENABLED }} + ${'next'} | ${{ after: pageInfo.endCursor, first: GRAPHQL_PAGE_SIZE_METADATA_ENABLED }} + `('$event event triggers correct page request', async ({ event, expected }) => { + const resolver = jest.fn().mockResolvedValue(graphQLImageListMock); + const detailsResolver = jest + .fn() + .mockResolvedValue(graphQLProjectImageRepositoriesDetailsMock); + const config = { + isMetadataDatabaseEnabled: true, + isGroupPage: false, + }; + + mountComponent({ resolver, detailsResolver, config }); + fireFirstSortUpdate(); + await waitForApolloRequestRender(); + + findPersistedPagination().vm.$emit(event); + await waitForPromises(); + + expect(resolver).toHaveBeenCalledWith(expect.objectContaining(expected)); + expect(detailsResolver).toHaveBeenCalledWith(expect.objectContaining(expected)); + }); + + it.each` + cursor | expected + ${{ before: pageInfo.startCursor }} | ${{ sort: 'UPDATED_DESC', before: pageInfo.startCursor, first: null, last: GRAPHQL_PAGE_SIZE_METADATA_ENABLED }} + ${{ after: pageInfo.endCursor }} | ${{ sort: 'UPDATED_DESC', after: pageInfo.endCursor, first: GRAPHQL_PAGE_SIZE_METADATA_ENABLED }} + `( + 'calls resolver correctly when persisted search returns $cursor', + async ({ cursor, expected }) => { + const resolver = jest.fn().mockResolvedValue(graphQLImageListMock); + const detailsResolver = jest + .fn() + .mockResolvedValue(graphQLProjectImageRepositoriesDetailsMock); + const config = { + isMetadataDatabaseEnabled: true, + isGroupPage: false, + }; + + mountComponent({ resolver, detailsResolver, config }); + + findPersistedSearch().vm.$emit('update', { + sort: 'UPDATED_DESC', + filters: [], + pageInfo: cursor, + }); + await waitForApolloRequestRender(); + + expect(resolver).toHaveBeenCalledWith(expect.objectContaining(expected)); + expect(detailsResolver).toHaveBeenCalledWith(expect.objectContaining(expected)); + }, + ); + }); }); }); diff --git a/spec/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_namespace_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_namespace_id_spec.rb new file mode 100644 index 00000000000..8cf90e02a52 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_namespace_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillComplianceFrameworkSecurityPoliciesNamespaceId, + feature_category: :security_policy_management, + schema: 20240722095916 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :compliance_framework_security_policies } + let(:backfill_column) { :namespace_id } + let(:backfill_via_table) { :security_orchestration_policy_configurations } + let(:backfill_via_column) { :namespace_id } + let(:backfill_via_foreign_key) { :policy_configuration_id } + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_project_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_project_id_spec.rb new file mode 100644 index 00000000000..e1ef60f0638 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_compliance_framework_security_policies_project_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillComplianceFrameworkSecurityPoliciesProjectId, + feature_category: :security_policy_management, + schema: 20240722095911 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :compliance_framework_security_policies } + let(:backfill_column) { :project_id } + let(:backfill_via_table) { :security_orchestration_policy_configurations } + let(:backfill_via_column) { :project_id } + let(:backfill_via_foreign_key) { :policy_configuration_id } + end +end diff --git a/spec/lib/gitlab/background_migration/deduplicate_lfs_objects_projects_spec.rb b/spec/lib/gitlab/background_migration/deduplicate_lfs_objects_projects_spec.rb new file mode 100644 index 00000000000..51fb55ea520 --- /dev/null +++ b/spec/lib/gitlab/background_migration/deduplicate_lfs_objects_projects_spec.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DeduplicateLfsObjectsProjects, feature_category: :source_code_management do + let(:namespace_table) { table(:namespaces) } + let(:projects_table) { table(:projects) } + let(:lfs_objects_table) { table(:lfs_objects) } + let(:lfs_objects_projects_table) { table(:lfs_objects_projects) } + let(:lfs_object_size) { 20 } + + let(:namespace1) { namespace_table.create!(name: 'namespace1', path: 'namespace1') } + let(:namespace2) { namespace_table.create!(name: 'namespace2', path: 'namespace2') } + let(:namespace3) { namespace_table.create!(name: 'namespace3', path: 'namespace3') } + + let(:project1) { projects_table.create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:project2) { projects_table.create!(namespace_id: namespace2.id, project_namespace_id: namespace2.id) } + let(:project3) { projects_table.create!(namespace_id: namespace3.id, project_namespace_id: namespace3.id) } + + let(:lfs_object1) do + lfs_objects_table.create!( + oid: 'f2b0a1e7550e9b718dafc9b525a04879a766de62e4fbdfc46593d47f7ab74636', + size: lfs_object_size + ) + end + + let(:lfs_object2) do + lfs_objects_table.create!( + oid: '004409da2260f89ceaf6d7cd13cbdfdeeeb43b6c1705299e96efbaa659805785', + size: lfs_object_size + ) + end + + let(:lfs_object3) do + lfs_objects_table.create!( + oid: 'c4c65374aa473c94e81810fbb0adb861292661eeb677a197a19a18e52e8eab9c', + size: lfs_object_size + ) + end + + let(:lfs_object4) do + lfs_objects_table.create!( + oid: '7736b17f9fff13bb54b10738acc03daa6c0867b22d29a5f9430cd97a47ebb6a4', + size: lfs_object_size + ) + end + + let(:lfs_object5) do + lfs_objects_table.create!( + oid: '96f74c6fe7a2979eefb9ec74a5dfc6888fb25543cf99b77586b79afea1da6f97', + size: lfs_object_size + ) + end + + let(:lfs_object6) do + lfs_objects_table.create!( + oid: '47997ea7ecff33be61e3ca1cc287ee72a2125161518f1a169f2893a5a82e9d95', + size: lfs_object_size + ) + end + + let!(:duplicated_lfs_objects_project1) do + lfs_objects_projects_table.create!(project_id: project1.id, lfs_object_id: lfs_object1.id, repository_type: 0) + end + + let!(:lfs_objects_project2) do + lfs_objects_projects_table.create!(project_id: project1.id, lfs_object_id: lfs_object1.id, repository_type: 0) + end + + let!(:duplicated_lfs_objects_project3) do + lfs_objects_projects_table.create!(project_id: project2.id, lfs_object_id: lfs_object4.id, repository_type: 0) + end + + let!(:duplicated_lfs_objects_project4) do + lfs_objects_projects_table.create!(project_id: project2.id, lfs_object_id: lfs_object4.id, repository_type: 0) + end + + let!(:lfs_objects_project5) do + lfs_objects_projects_table.create!(project_id: project2.id, lfs_object_id: lfs_object4.id, repository_type: 0) + end + + let!(:lfs_objects_project6) do + lfs_objects_projects_table.create!(project_id: project3.id, lfs_object_id: lfs_object6.id, repository_type: 1) + end + + let(:migration_attrs) do + { + start_id: projects_table.minimum(:id), + end_id: projects_table.maximum(:id), + batch_table: :lfs_objects_projects, + batch_column: :project_id, + sub_batch_size: 3, + pause_ms: 100, + connection: ApplicationRecord.connection + } + end + + subject(:migration) { described_class.new(**migration_attrs) } + + describe '#perform' do + context 'with duplicates' do + it 'deduplicates lfs_objects_projects by lfs_object_id, repository_type and project_id' do + expect { migration.perform }.to change { lfs_objects_projects_table.count }.from(6).to(3) + + expect(lfs_objects_projects_table.all) + .to contain_exactly(lfs_objects_project2, lfs_objects_project5, lfs_objects_project6) + end + end + + context 'without duplicates' do + it 'does not delete any records' do + [ + duplicated_lfs_objects_project1, + duplicated_lfs_objects_project3, + duplicated_lfs_objects_project4 + ].each(&:destroy) + + expect { migration.perform }.not_to change { lfs_objects_table.count } + end + end + end +end diff --git a/spec/lib/gitlab/database/dictionary_spec.rb b/spec/lib/gitlab/database/dictionary_spec.rb index 2e22b9a3510..289ad59800b 100644 --- a/spec/lib/gitlab/database/dictionary_spec.rb +++ b/spec/lib/gitlab/database/dictionary_spec.rb @@ -75,7 +75,7 @@ RSpec.describe Gitlab::Database::Dictionary, feature_category: :database do it 'returns an array of entries having desired sharding key migration job' do entries = dictionary.find_all_having_desired_sharding_key_migration_job expect(entries).to all(be_instance_of(Gitlab::Database::Dictionary::Entry)) - expect(entries).to all(have_attributes(desired_sharding_key_migration_job_name: String)) + expect(entries.map(&:desired_sharding_key_migration_job_name)).to all(be_present) end end diff --git a/spec/migrations/20240722095915_queue_backfill_compliance_framework_security_policies_project_id_spec.rb b/spec/migrations/20240722095915_queue_backfill_compliance_framework_security_policies_project_id_spec.rb new file mode 100644 index 00000000000..f9cd4fad99d --- /dev/null +++ b/spec/migrations/20240722095915_queue_backfill_compliance_framework_security_policies_project_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillComplianceFrameworkSecurityPoliciesProjectId, feature_category: :security_policy_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :compliance_framework_security_policies, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :project_id, + :security_orchestration_policy_configurations, + :project_id, + :policy_configuration_id + ] + ) + } + end + end +end diff --git a/spec/migrations/20240722095920_queue_backfill_compliance_framework_security_policies_namespace_id_spec.rb b/spec/migrations/20240722095920_queue_backfill_compliance_framework_security_policies_namespace_id_spec.rb new file mode 100644 index 00000000000..d640116f42f --- /dev/null +++ b/spec/migrations/20240722095920_queue_backfill_compliance_framework_security_policies_namespace_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillComplianceFrameworkSecurityPoliciesNamespaceId, feature_category: :security_policy_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :compliance_framework_security_policies, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :namespace_id, + :security_orchestration_policy_configurations, + :namespace_id, + :policy_configuration_id + ] + ) + } + end + end +end diff --git a/spec/migrations/20240808125149_queue_deduplicate_lfs_objects_projects_spec.rb b/spec/migrations/20240808125149_queue_deduplicate_lfs_objects_projects_spec.rb new file mode 100644 index 00000000000..930361c7774 --- /dev/null +++ b/spec/migrations/20240808125149_queue_deduplicate_lfs_objects_projects_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueDeduplicateLfsObjectsProjects, feature_category: :source_code_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :lfs_objects_projects, + column_name: :id, + batch_class_name: described_class::BATCH_CLASS_NAME, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end diff --git a/workhorse/_support/lint_last_known_acceptable_go1.21.txt b/workhorse/_support/lint_last_known_acceptable_go1.21.txt index 39d7c7e7fea..0911364d46f 100644 --- a/workhorse/_support/lint_last_known_acceptable_go1.21.txt +++ b/workhorse/_support/lint_last_known_acceptable_go1.21.txt @@ -3,8 +3,6 @@ cmd/gitlab-resize-image/png/reader.go:1:1: package-comments: should have a packa cmd/gitlab-resize-image/png/reader.go:26:1: exported: exported function NewReader should have comment or be unexported (revive) cmd/gitlab-resize-image/png/reader.go:78:17: var-declaration: should omit type []byte from declaration of var magicBytes; it will be inferred from the right-hand side (revive) cmd/gitlab-workhorse/config_test.go:191: cmd/gitlab-workhorse/config_test.go:191: Line contains TODO/BUG/FIXME/NOTE/OPTIMIZE/HACK: "TODO this is meant to be 50*time.Second ..." (godox) -cmd/gitlab-workhorse/jobs_test.go:37:29: response body must be closed (bodyclose) -cmd/gitlab-workhorse/jobs_test.go:42:29: response body must be closed (bodyclose) cmd/gitlab-workhorse/listener.go:26:16: G402: TLS MinVersion too low. (gosec) cmd/gitlab-workhorse/main.go:9:2: G108: Profiling endpoint is automatically exposed on /debug/pprof (gosec) cmd/gitlab-workhorse/main.go:73: Function 'buildConfig' has too many statements (63 > 40) (funlen) @@ -22,30 +20,16 @@ cmd/gitlab-workhorse/main.go:233:5: shadow: declaration of "err" shadows declara cmd/gitlab-workhorse/main.go:241:26: Error return value of `accessCloser.Close` is not checked (errcheck) cmd/gitlab-workhorse/main.go:265:10: G112: Potential Slowloris Attack because ReadHeaderTimeout is not configured in the http.Server (gosec) cmd/gitlab-workhorse/main_test.go:60:2: exitAfterDefer: os.Exit will exit, and `defer gitaly.CloseConnections()` will not run (gocritic) -cmd/gitlab-workhorse/main_test.go:107:24: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:143:24: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:156:66: wrapperFunc: use http.NotFoundHandler method in `http.HandlerFunc(http.NotFound)` (gocritic) -cmd/gitlab-workhorse/main_test.go:163:23: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:186:24: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:212:25: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:243:23: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:246:25: unnecessary conversion (unconvert) -cmd/gitlab-workhorse/main_test.go:416:38: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:431:38: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:457:3: bool-compare: use assert.True (testifylint) -cmd/gitlab-workhorse/main_test.go:485:40: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:509:23: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:519:3: ifElseChain: rewrite if-else to switch statement (gocritic) -cmd/gitlab-workhorse/main_test.go:575:21: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:578:3: var-naming: var requestIds should be requestIDs (revive) -cmd/gitlab-workhorse/main_test.go:579:3: len: use require.Len (testifylint) -cmd/gitlab-workhorse/main_test.go:651:4: var-naming: var propagatedRequestId should be propagatedRequestID (revive) -cmd/gitlab-workhorse/main_test.go:658:22: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:659:4: var-naming: var requestIds should be requestIDs (revive) -cmd/gitlab-workhorse/main_test.go:662:4: len: use require.Len (testifylint) -cmd/gitlab-workhorse/main_test.go:871:25: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:896:25: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:937:4: var-naming: var originResourceUrl should be originResourceURL (revive) +cmd/gitlab-workhorse/main_test.go:158:66: wrapperFunc: use http.NotFoundHandler method in `http.HandlerFunc(http.NotFound)` (gocritic) +cmd/gitlab-workhorse/main_test.go:252:25: unnecessary conversion (unconvert) +cmd/gitlab-workhorse/main_test.go:467:3: bool-compare: use assert.True (testifylint) +cmd/gitlab-workhorse/main_test.go:532:3: ifElseChain: rewrite if-else to switch statement (gocritic) +cmd/gitlab-workhorse/main_test.go:592:3: var-naming: var requestIds should be requestIDs (revive) +cmd/gitlab-workhorse/main_test.go:593:3: len: use require.Len (testifylint) +cmd/gitlab-workhorse/main_test.go:665:4: var-naming: var propagatedRequestId should be propagatedRequestID (revive) +cmd/gitlab-workhorse/main_test.go:674:4: var-naming: var requestIds should be requestIDs (revive) +cmd/gitlab-workhorse/main_test.go:677:4: len: use require.Len (testifylint) +cmd/gitlab-workhorse/main_test.go:954:4: var-naming: var originResourceUrl should be originResourceURL (revive) cmd/gitlab-workhorse/proxy_test.go:55:9: shadow: declaration of "err" shadows declaration at line 36 (govet) cmd/gitlab-workhorse/proxy_test.go:77:6: var-naming: var tsUrl should be tsURL (revive) cmd/gitlab-workhorse/proxy_test.go:87:6: shadow: declaration of "err" shadows declaration at line 78 (govet) @@ -107,8 +91,7 @@ internal/dependencyproxy/dependencyproxy.go:78: Function 'Inject' is too long (7 internal/dependencyproxy/dependencyproxy.go:116:32: `cancelled` is a misspelling of `canceled` (misspell) internal/dependencyproxy/dependencyproxy_test.go:376:4: go-require: do not use assert.FailNow in http handlers (testifylint) internal/dependencyproxy/dependencyproxy_test.go:403:33: `artifically` is a misspelling of `artificially` (misspell) -internal/dependencyproxy/dependencyproxy_test.go:415:27: response body must be closed (bodyclose) -internal/dependencyproxy/dependencyproxy_test.go:426: internal/dependencyproxy/dependencyproxy_test.go:426: Line contains TODO/BUG/FIXME/NOTE/OPTIMIZE/HACK: "note that the timeout duration here is s..." (godox) +internal/dependencyproxy/dependencyproxy_test.go:428: internal/dependencyproxy/dependencyproxy_test.go:428: Line contains TODO/BUG/FIXME/NOTE/OPTIMIZE/HACK: "note that the timeout duration here is s..." (godox) internal/git/archive.go:35:2: var-naming: struct field CommitId should be CommitID (revive) internal/git/archive.go:43:2: exported: exported var SendArchive should have comment or be unexported (revive) internal/git/archive.go:53: Function 'Inject' has too many statements (47 > 40) (funlen) @@ -177,15 +160,6 @@ internal/imageresizer/image_resizer.go:350:17: Error return value of `res.Body.C internal/imageresizer/image_resizer.go:356:15: G304: Potential file inclusion via variable (gosec) internal/imageresizer/image_resizer.go:363:13: Error return value of `file.Close` is not checked (errcheck) internal/imageresizer/image_resizer_caching.go:6:1: ST1000: at least one file in a package should have a package comment (stylecheck) -internal/imageresizer/image_resizer_test.go:76:30: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:93:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:108:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:120:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:130:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:140:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:148:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:193:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:216:28: response body must be closed (bodyclose) internal/lsif_transformer/parser/docs.go:36:2: var-naming: struct field RangeIds should be RangeIDs (revive) internal/lsif_transformer/parser/parser.go:90:1: exported: exported method Parser.Close should have comment or be unexported (revive) internal/lsif_transformer/parser/ranges.go:35:2: var-naming: struct field RangeIds should be RangeIDs (revive) diff --git a/workhorse/_support/lint_last_known_acceptable_go1.22.txt b/workhorse/_support/lint_last_known_acceptable_go1.22.txt index 36d0e30dcd7..98734858876 100644 --- a/workhorse/_support/lint_last_known_acceptable_go1.22.txt +++ b/workhorse/_support/lint_last_known_acceptable_go1.22.txt @@ -3,8 +3,6 @@ cmd/gitlab-resize-image/png/reader.go:1:1: package-comments: should have a packa cmd/gitlab-resize-image/png/reader.go:26:1: exported: exported function NewReader should have comment or be unexported (revive) cmd/gitlab-resize-image/png/reader.go:78:17: var-declaration: should omit type []byte from declaration of var magicBytes; it will be inferred from the right-hand side (revive) cmd/gitlab-workhorse/config_test.go:191: cmd/gitlab-workhorse/config_test.go:191: Line contains TODO/BUG/FIXME/NOTE/OPTIMIZE/HACK: "TODO this is meant to be 50*time.Second ..." (godox) -cmd/gitlab-workhorse/jobs_test.go:37:29: response body must be closed (bodyclose) -cmd/gitlab-workhorse/jobs_test.go:42:29: response body must be closed (bodyclose) cmd/gitlab-workhorse/listener.go:26:16: G402: TLS MinVersion too low. (gosec) cmd/gitlab-workhorse/main.go:9:2: G108: Profiling endpoint is automatically exposed on /debug/pprof (gosec) cmd/gitlab-workhorse/main.go:73: Function 'buildConfig' has too many statements (63 > 40) (funlen) @@ -22,30 +20,16 @@ cmd/gitlab-workhorse/main.go:233:5: shadow: declaration of "err" shadows declara cmd/gitlab-workhorse/main.go:241:26: Error return value of `accessCloser.Close` is not checked (errcheck) cmd/gitlab-workhorse/main.go:265:10: G112: Potential Slowloris Attack because ReadHeaderTimeout is not configured in the http.Server (gosec) cmd/gitlab-workhorse/main_test.go:60:2: exitAfterDefer: os.Exit will exit, and `defer gitaly.CloseConnections()` will not run (gocritic) -cmd/gitlab-workhorse/main_test.go:107:24: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:143:24: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:156:66: wrapperFunc: use http.NotFoundHandler method in `http.HandlerFunc(http.NotFound)` (gocritic) -cmd/gitlab-workhorse/main_test.go:163:23: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:186:24: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:212:25: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:243:23: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:246:25: unnecessary conversion (unconvert) -cmd/gitlab-workhorse/main_test.go:416:38: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:431:38: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:457:3: bool-compare: use assert.True (testifylint) -cmd/gitlab-workhorse/main_test.go:485:40: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:509:23: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:519:3: ifElseChain: rewrite if-else to switch statement (gocritic) -cmd/gitlab-workhorse/main_test.go:575:21: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:578:3: var-naming: var requestIds should be requestIDs (revive) -cmd/gitlab-workhorse/main_test.go:579:3: len: use require.Len (testifylint) -cmd/gitlab-workhorse/main_test.go:651:4: var-naming: var propagatedRequestId should be propagatedRequestID (revive) -cmd/gitlab-workhorse/main_test.go:658:22: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:659:4: var-naming: var requestIds should be requestIDs (revive) -cmd/gitlab-workhorse/main_test.go:662:4: len: use require.Len (testifylint) -cmd/gitlab-workhorse/main_test.go:871:25: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:896:25: response body must be closed (bodyclose) -cmd/gitlab-workhorse/main_test.go:937:4: var-naming: var originResourceUrl should be originResourceURL (revive) +cmd/gitlab-workhorse/main_test.go:158:66: wrapperFunc: use http.NotFoundHandler method in `http.HandlerFunc(http.NotFound)` (gocritic) +cmd/gitlab-workhorse/main_test.go:252:25: unnecessary conversion (unconvert) +cmd/gitlab-workhorse/main_test.go:467:3: bool-compare: use assert.True (testifylint) +cmd/gitlab-workhorse/main_test.go:532:3: ifElseChain: rewrite if-else to switch statement (gocritic) +cmd/gitlab-workhorse/main_test.go:592:3: var-naming: var requestIds should be requestIDs (revive) +cmd/gitlab-workhorse/main_test.go:593:3: len: use require.Len (testifylint) +cmd/gitlab-workhorse/main_test.go:665:4: var-naming: var propagatedRequestId should be propagatedRequestID (revive) +cmd/gitlab-workhorse/main_test.go:674:4: var-naming: var requestIds should be requestIDs (revive) +cmd/gitlab-workhorse/main_test.go:677:4: len: use require.Len (testifylint) +cmd/gitlab-workhorse/main_test.go:954:4: var-naming: var originResourceUrl should be originResourceURL (revive) cmd/gitlab-workhorse/proxy_test.go:55:9: shadow: declaration of "err" shadows declaration at line 36 (govet) cmd/gitlab-workhorse/proxy_test.go:77:6: var-naming: var tsUrl should be tsURL (revive) cmd/gitlab-workhorse/proxy_test.go:87:6: shadow: declaration of "err" shadows declaration at line 78 (govet) @@ -107,8 +91,7 @@ internal/dependencyproxy/dependencyproxy.go:78: Function 'Inject' is too long (7 internal/dependencyproxy/dependencyproxy.go:116:32: `cancelled` is a misspelling of `canceled` (misspell) internal/dependencyproxy/dependencyproxy_test.go:376:4: go-require: do not use assert.FailNow in http handlers (testifylint) internal/dependencyproxy/dependencyproxy_test.go:403:33: `artifically` is a misspelling of `artificially` (misspell) -internal/dependencyproxy/dependencyproxy_test.go:415:27: response body must be closed (bodyclose) -internal/dependencyproxy/dependencyproxy_test.go:426: internal/dependencyproxy/dependencyproxy_test.go:426: Line contains TODO/BUG/FIXME/NOTE/OPTIMIZE/HACK: "note that the timeout duration here is s..." (godox) +internal/dependencyproxy/dependencyproxy_test.go:428: internal/dependencyproxy/dependencyproxy_test.go:428: Line contains TODO/BUG/FIXME/NOTE/OPTIMIZE/HACK: "note that the timeout duration here is s..." (godox) internal/git/archive.go:35:2: var-naming: struct field CommitId should be CommitID (revive) internal/git/archive.go:43:2: exported: exported var SendArchive should have comment or be unexported (revive) internal/git/archive.go:53: Function 'Inject' has too many statements (47 > 40) (funlen) @@ -177,15 +160,6 @@ internal/imageresizer/image_resizer.go:350:17: Error return value of `res.Body.C internal/imageresizer/image_resizer.go:356:15: G304: Potential file inclusion via variable (gosec) internal/imageresizer/image_resizer.go:363:13: Error return value of `file.Close` is not checked (errcheck) internal/imageresizer/image_resizer_caching.go:6:1: ST1000: at least one file in a package should have a package comment (stylecheck) -internal/imageresizer/image_resizer_test.go:76:30: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:93:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:108:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:120:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:130:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:140:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:148:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:193:28: response body must be closed (bodyclose) -internal/imageresizer/image_resizer_test.go:216:28: response body must be closed (bodyclose) internal/lsif_transformer/parser/docs.go:36:2: var-naming: struct field RangeIds should be RangeIDs (revive) internal/lsif_transformer/parser/parser.go:90:1: exported: exported method Parser.Close should have comment or be unexported (revive) internal/lsif_transformer/parser/ranges.go:35:2: var-naming: struct field RangeIds should be RangeIDs (revive) diff --git a/workhorse/cmd/gitlab-workhorse/jobs_test.go b/workhorse/cmd/gitlab-workhorse/jobs_test.go index 7846788bae4..2a78e03dd83 100644 --- a/workhorse/cmd/gitlab-workhorse/jobs_test.go +++ b/workhorse/cmd/gitlab-workhorse/jobs_test.go @@ -35,11 +35,13 @@ func testJobsLongPolling(t *testing.T, pollingDuration time.Duration, requestJob func testJobsLongPollingEndpointDisabled(t *testing.T, requestJob requestJobFunction) { resp := testJobsLongPolling(t, 0, requestJob) + defer resp.Body.Close() require.NotEqual(t, "yes", resp.Header.Get("Gitlab-Ci-Builds-Polling")) } func testJobsLongPollingEndpoint(t *testing.T, requestJob requestJobFunction) { resp := testJobsLongPolling(t, time.Minute, requestJob) + defer resp.Body.Close() require.Equal(t, "yes", resp.Header.Get("Gitlab-Ci-Builds-Polling")) } diff --git a/workhorse/cmd/gitlab-workhorse/main_test.go b/workhorse/cmd/gitlab-workhorse/main_test.go index 763fd668319..08ccb60859c 100644 --- a/workhorse/cmd/gitlab-workhorse/main_test.go +++ b/workhorse/cmd/gitlab-workhorse/main_test.go @@ -105,6 +105,7 @@ func TestRegularProjectsAPI(t *testing.T) { "/api/v3/projects/foo%2Fbar%2Fbaz%2Fqux/repository/not/special", } { resp, body := httpGet(t, ws.URL+resource, nil) + defer resp.Body.Close() require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resource) require.Equal(t, apiResponse, body, "GET %q: response body", resource) @@ -141,6 +142,7 @@ func TestAllowedStaticFile(t *testing.T) { "/static file.txt", } { resp, body := httpGet(t, ws.URL+resource, nil) + defer resp.Body.Close() require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resource) require.Equal(t, content, body, "GET %q: response body", resource) @@ -161,6 +163,7 @@ func TestStaticFileRelativeURL(t *testing.T) { resource := "/my-relative-url/static.txt" resp, body := httpGet(t, ws.URL+resource, nil) + defer resp.Body.Close() require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resource) require.Equal(t, content, body, "GET %q: response body", resource) @@ -184,6 +187,7 @@ func TestAllowedPublicUploadsFile(t *testing.T) { "/uploads/static file.txt", } { resp, body := httpGet(t, ws.URL+resource, nil) + defer resp.Body.Close() require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resource) require.Equal(t, content, body, "GET %q: response body", resource) @@ -210,6 +214,7 @@ func TestDeniedPublicUploadsFile(t *testing.T) { } { t.Run(resource, func(t *testing.T) { resp, body := httpGet(t, ws.URL+resource, nil) + defer resp.Body.Close() require.Equal(t, 404, resp.StatusCode, "GET %q: status code", resource) require.Equal(t, "", body, "GET %q: response body", resource) @@ -241,6 +246,7 @@ This is a static error page for code 499 resourcePath := "/error-499" resp, body := httpGet(t, ws.URL+resourcePath, nil) + defer resp.Body.Close() require.Equal(t, 499, resp.StatusCode, "GET %q: status code", resourcePath) require.Equal(t, string(errorPageBody), body, "GET %q: response body", resourcePath) @@ -415,6 +421,7 @@ func TestArtifactsGetSingleFile(t *testing.T) { resp, body, err := doSendDataRequest(t, resourcePath, "artifacts-entry", jsonParams) require.NoError(t, err) + defer resp.Body.Close() require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resourcePath) require.Equal(t, fileContents, string(body), "GET %q: response body", resourcePath) @@ -429,7 +436,10 @@ func TestImageResizing(t *testing.T) { resourcePath := "/uploads/-/system/user/avatar/123/avatar.png?width=40" resp, body, err := doSendDataRequest(t, resourcePath, "send-scaled-img", jsonParams) + require.NoError(t, err, "send resize request") + defer resp.Body.Close() + require.Equal(t, 200, resp.StatusCode, "GET %q: body: %s", resourcePath, body) img, err := png.Decode(bytes.NewReader(body)) @@ -485,6 +495,8 @@ func TestSendURLForArtifacts(t *testing.T) { resp, body, err := doSendDataRequest(t, resourcePath, "send-url", jsonParams) require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode, "GET %q: status code", resourcePath) require.Equal(t, int64(tc.contentLength), resp.ContentLength, "GET %q: Content-Length", resourcePath) require.Equal(t, tc.transferEncoding, resp.TransferEncoding, "GET %q: Transfer-Encoding", resourcePath) @@ -507,6 +519,7 @@ func TestApiContentTypeBlock(t *testing.T) { resourcePath := "/something" resp, body := httpGet(t, ws.URL+resourcePath, nil) + defer resp.Body.Close() require.Equal(t, 500, resp.StatusCode, "GET %q: status code", resourcePath) require.NotContains(t, wrongResponse, body, "GET %q: response body", resourcePath) @@ -573,6 +586,7 @@ func TestCorrelationIdHeader(t *testing.T) { "/api/v3/projects/123/repository/not/special", } { resp, _ := httpGet(t, ws.URL+resource, nil) + defer resp.Body.Close() require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resource) requestIds := resp.Header["X-Request-Id"] @@ -656,6 +670,7 @@ func TestPropagateCorrelationIdHeader(t *testing.T) { } resp, _ := httpGet(t, ws.URL+resource, headers) + defer resp.Body.Close() requestIds := resp.Header["X-Request-Id"] require.Equal(t, 200, resp.StatusCode, "GET %q: status code", resource) @@ -869,6 +884,7 @@ This is a static error page for code 503 } { t.Run(resource, func(t *testing.T) { resp, body := httpGet(t, ws.URL+resource, nil) + defer resp.Body.Close() require.Equal(t, 503, resp.StatusCode, "status code") require.Equal(t, apiResponse, body, "response body") @@ -894,6 +910,7 @@ func TestHealthChecksUnreachable(t *testing.T) { for _, tc := range testCases { t.Run(tc.path, func(t *testing.T) { resp, body := httpGet(t, ws.URL+tc.path, nil) + defer resp.Body.Close() require.Equal(t, 502, resp.StatusCode, "status code") require.Equal(t, tc.responseType, resp.Header.Get("Content-Type"), "content-type") diff --git a/workhorse/internal/dependencyproxy/dependencyproxy_test.go b/workhorse/internal/dependencyproxy/dependencyproxy_test.go index ccd5bd7aa5e..2e85372374c 100644 --- a/workhorse/internal/dependencyproxy/dependencyproxy_test.go +++ b/workhorse/internal/dependencyproxy/dependencyproxy_test.go @@ -415,6 +415,8 @@ func TestLongUploadRequest(t *testing.T) { res, err := rt.RoundTrip(r) assert.NoError(t, err, "RoundTripper should not receive an error") + defer res.Body.Close() + assert.Equal(t, http.StatusOK, res.StatusCode, "RoundTripper should receive a 200 status code") w.WriteHeader(res.StatusCode) } diff --git a/workhorse/internal/imageresizer/image_resizer_test.go b/workhorse/internal/imageresizer/image_resizer_test.go index f16df77f002..7edcac34e06 100644 --- a/workhorse/internal/imageresizer/image_resizer_test.go +++ b/workhorse/internal/imageresizer/image_resizer_test.go @@ -74,6 +74,7 @@ func TestRequestScaledImageFromPath(t *testing.T) { params := resizeParams{Location: tc.imagePath, ContentType: tc.contentType, Width: 64} resp := requestScaledImage(t, nil, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) bounds := imageFromResponse(t, resp).Bounds() @@ -91,6 +92,7 @@ func TestRequestScaledImageWithConditionalGetAndImageNotChanged(t *testing.T) { header.Set("If-Modified-Since", httpTimeStr(clientTime)) resp := requestScaledImage(t, header, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusNotModified, resp.StatusCode) require.Equal(t, httpTimeStr(testImageLastModified(t)), resp.Header.Get("Last-Modified")) require.Empty(t, resp.Header.Get("Content-Type")) @@ -106,6 +108,7 @@ func TestRequestScaledImageWithConditionalGetAndImageChanged(t *testing.T) { header.Set("If-Modified-Since", httpTimeStr(clientTime)) resp := requestScaledImage(t, header, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) require.Equal(t, httpTimeStr(testImageLastModified(t)), resp.Header.Get("Last-Modified")) } @@ -118,6 +121,7 @@ func TestRequestScaledImageWithConditionalGetAndInvalidClientTime(t *testing.T) header.Set("If-Modified-Since", "0") resp := requestScaledImage(t, header, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) require.Equal(t, httpTimeStr(testImageLastModified(t)), resp.Header.Get("Last-Modified")) } @@ -128,6 +132,7 @@ func TestServeOriginalImageWhenSourceImageTooLarge(t *testing.T) { params := resizeParams{Location: imagePath, ContentType: "image/png", Width: 64} resp := requestScaledImage(t, nil, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) img := imageFromResponse(t, resp) @@ -138,6 +143,7 @@ func TestFailFastOnOpenStreamFailure(t *testing.T) { cfg := config.DefaultImageResizerConfig params := resizeParams{Location: "does_not_exist.png", ContentType: "image/png", Width: 64} resp := requestScaledImage(t, nil, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusInternalServerError, resp.StatusCode) } @@ -146,6 +152,7 @@ func TestIgnoreContentTypeMismatchIfImageFormatIsAllowed(t *testing.T) { cfg := config.DefaultImageResizerConfig params := resizeParams{Location: imagePath, ContentType: "image/jpeg", Width: 64} resp := requestScaledImage(t, nil, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) bounds := imageFromResponse(t, resp).Bounds() @@ -191,6 +198,7 @@ func TestServeOriginalImageWhenSourceImageFormatIsNotAllowed(t *testing.T) { params := resizeParams{Location: svgImagePath, ContentType: "image/png", Width: 64} resp := requestScaledImage(t, nil, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) responseData, err := io.ReadAll(resp.Body) @@ -214,6 +222,7 @@ func TestServeOriginalImageWhenSourceImageIsTooSmall(t *testing.T) { params := resizeParams{Location: img.Name(), ContentType: "image/png", Width: 64} resp := requestScaledImage(t, nil, params, cfg) + defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) responseData, err := io.ReadAll(resp.Body)