From 1dace033aacf8f80a31117341b36f9efaee1c210 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 30 Apr 2024 09:15:33 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .../graphql/resolver_method_length.yml | 2 - .rubocop_todo/layout/space_inside_parens.yml | 1 - .../lint/ambiguous_operator_precedence.yml | 10 - .rubocop_todo/rspec/context_wording.yml | 1 - .rubocop_todo/rspec/feature_category.yml | 1 - .rubocop_todo/rspec/named_subject.yml | 1 - .../style/numeric_literal_prefix.yml | 1 - .../components/clusters_empty_state.vue | 4 +- .../components/add_exclusions_drawer.vue | 84 ++ .../components/exclusions_list.vue | 126 ++ .../components/exclusions_list_item.vue | 46 + .../components/exclusions_tabs.vue | 39 + .../remove_exclusion_confirmation_modal.vue | 58 + .../admin/integrations/overrides/index.js | 30 + .../javascripts/tracking/internal_events.js | 31 +- .../incubation/incubation_alert.vue | 4 +- .../components/list_selector/constants.js | 7 + .../components/list_selector/index.vue | 79 +- .../components/list_selector/project_item.vue | 56 + .../shared/work_item_link_child_contents.vue | 1 - .../ml/models_index_component.html.haml | 1 - .../projects/ml/models_index_component.rb | 68 -- .../ml/show_ml_model_component.html.haml | 1 - .../projects/ml/show_ml_model_component.rb | 47 - .../show_ml_model_version_component.html.haml | 1 - .../ml/show_ml_model_version_component.rb | 41 - app/graphql/types/ci/runner_type.rb | 4 +- app/models/ci/build_metadata.rb | 4 - app/models/ci/build_name.rb | 4 - app/models/ci/job_annotation.rb | 4 - app/models/ci/job_artifact.rb | 4 - app/models/ci/pipeline.rb | 4 - app/models/ci/pipeline_variable.rb | 4 - app/models/ci/runner_manager_build.rb | 4 - app/models/ci/stage.rb | 4 - app/models/commit_status.rb | 4 - app/models/concerns/ci/has_status.rb | 1 + ...ed_private_group_accessibility_assigner.rb | 2 +- app/views/shared/integrations/_tabs.html.haml | 3 +- .../shared/integrations/overrides.html.haml | 6 +- .../import_export/relation_import_worker.rb | 25 +- .../events/20230321151607_github_create.yml | 2 +- ...ubGistsImport__ImportGistWorker_create.yml | 2 +- .../beta/beyond_identity_exclusions.yml | 9 + ...15-9-deprecate-legacy-requirements-iid.yml | 4 +- doc/api/graphql/reference/index.md | 1 + doc/ci/yaml/index.md | 2 +- .../quick_start.md | 145 +++ doc/update/deprecations.md | 28 +- .../checks/application_information_check.md | 24 + .../checks/authentication_token_check.md | 29 + .../checks/cleartext_authentication_check.md | 20 + .../api_security_testing/checks/cors_check.md | 20 + .../checks/dns_rebinding_check.md | 20 + .../checks/framework_debug_mode_check.md | 21 + .../checks/heartbleed_open_ssl_check.md | 22 + .../checks/html_injection_check.md | 28 + .../api_security_testing/checks/index.md | 140 +++ .../checks/insecure_http_methods_check.md | 22 + .../checks/json_hijacking_check.md | 20 + .../checks/json_injection_check.md | 20 + .../checks/open_redirect_check.md | 20 + .../checks/os_command_injection_check.md | 32 + .../checks/path_traversal_check.md | 30 + .../checks/sensitive_file_disclosure_check.md | 22 + .../sensitive_information_disclosure_check.md | 22 + .../checks/session_cookie_check.md | 27 + .../checks/shellshock_check.md | 20 + .../checks/sql_injection_check.md | 24 + .../checks/tls_server_configuration_check.md | 29 + .../checks/xml_external_entity_check.md | 20 + .../checks/xml_injection_check.md | 20 + .../customizing_analyzer_settings.md | 1014 ++++++++++++++++ .../configuration/enabling_the_analyzer.md | 937 +++++++++++++++ .../configuration/index.md | 15 + .../configuration/offline_configuration.md | 30 + .../configuration/overriding_analyzer_jobs.md | 21 + .../configuration/requirements.md | 114 ++ .../configuration/variables.md | 104 ++ ...t_api_postman_collection_edit_variable.png | Bin ..._api_postman_environment_edit_variable.png | Bin .../img/dast_api_postman_request_edit.png | Bin .../api_security_testing/index.md | 85 ++ .../api_security_testing/performance.md | 219 ++++ .../api_security_testing/troubleshooting.md | 302 +++++ .../checks/application_information_check.md | 27 +- .../checks/authentication_token_check.md | 32 +- .../checks/cleartext_authentication_check.md | 23 +- .../dast_api/checks/cors_check.md | 23 +- .../dast_api/checks/dns_rebinding_check.md | 23 +- .../checks/framework_debug_mode_check.md | 24 +- .../checks/heartbleed_open_ssl_check.md | 25 +- .../dast_api/checks/html_injection_check.md | 31 +- .../dast_api/checks/index.md | 143 +-- .../checks/insecure_http_methods_check.md | 25 +- .../dast_api/checks/json_hijacking_check.md | 23 +- .../dast_api/checks/json_injection_check.md | 23 +- .../dast_api/checks/open_redirect_check.md | 23 +- .../checks/os_command_injection_check.md | 35 +- .../dast_api/checks/path_traversal_check.md | 33 +- .../checks/sensitive_file_disclosure_check.md | 25 +- .../sensitive_information_disclosure_check.md | 25 +- .../dast_api/checks/session_cookie_check.md | 30 +- .../dast_api/checks/shellshock_check.md | 23 +- .../dast_api/checks/sql_injection_check.md | 27 +- .../checks/tls_server_configuration_check.md | 32 +- .../dast_api/checks/xml_external_entity.md | 23 +- .../dast_api/checks/xml_injection_check.md | 23 +- .../customizing_analyzer_settings.md | 1017 +---------------- .../configuration/enabling_the_analyzer.md | 940 +-------------- .../dast_api/configuration/index.md | 18 +- .../configuration/offline_configuration.md | 33 +- .../configuration/overriding_analyzer_jobs.md | 24 +- .../dast_api/configuration/requirements.md | 117 +- .../dast_api/configuration/variables.md | 107 +- .../application_security/dast_api/index.md | 88 +- .../dast_api/performance.md | 222 +--- .../dast_api/troubleshooting.md | 305 +---- .../dependency_proxy/index.md | 2 +- .../gitlab_patches/partitioning.rb | 4 - .../gitlab_patches/partitioning/base.rb | 8 - .../reflection/abstract_reflection.rb | 4 +- .../spec/support/models.rb | 4 - .../generate_list.rb | 3 +- .../paginators/azure.rb | 27 + lib/gitlab/i18n/po_linter.rb | 4 +- lib/gitlab/import_export/file_importer.rb | 55 +- .../import_export/project/relation_factory.rb | 2 +- lib/gitlab/memory/instrumentation.rb | 2 +- lib/gitlab/metrics/prometheus.rb | 2 +- lib/gitlab/middleware/rails_queue_duration.rb | 2 +- lib/gitlab/pagination/gitaly_keyset_pager.rb | 2 +- lib/gitlab/quick_actions/issuable_actions.rb | 2 +- lib/gitlab/relative_positioning.rb | 2 +- lib/gitlab/template_parser/parser.rb | 2 +- lib/gitlab/tree_summary.rb | 2 +- locale/gitlab.pot | 42 +- .../ml/models_index_component_spec.rb | 89 -- .../ml/show_ml_model_component_spec.rb | 63 - .../show_ml_model_version_component_spec.rb | 66 -- .../tracking_internal_events_helper.js | 40 + .../components/add_exclusions_drawer_spec.js | 99 ++ .../components/exclusions_list_item_spec.js | 59 + .../components/exclusions_list_spec.js | 111 ++ .../components/exclusions_tabs_spec.js | 39 + .../beyond_identity/components/mock_data.js | 4 + ...emove_exclusion_confirmation_modal_spec.js | 40 + .../frontend/tracking/internal_events_spec.js | 61 +- .../components/list_selector/index_spec.js | 96 +- .../list_selector/project_item_spec.js | 59 + .../import_export/file_importer_spec.rb | 117 +- spec/models/concerns/ci/has_status_spec.rb | 12 + spec/support/rspec_order_todo.yml | 1 - .../relation_import_worker_spec.rb | 8 + .../gitaly_integration_test.go | 43 +- workhorse/cmd/gitlab-workhorse/main_test.go | 4 + workhorse/internal/headers/content_headers.go | 1 + workhorse/internal/headers/headers.go | 5 +- workhorse/internal/headers/headers_test.go | 8 +- .../internal/rejectmethods/middleware.go | 1 + .../internal/rejectmethods/middleware_test.go | 3 + workhorse/internal/secret/jwt.go | 3 + workhorse/internal/secret/roundtripper.go | 2 +- workhorse/internal/secret/secret.go | 3 +- .../internal/upload/multipart_uploader.go | 6 + .../upload/object_storage_preparer.go | 2 + workhorse/internal/upload/preparer.go | 2 + workhorse/internal/upload/rewrite.go | 14 +- workhorse/internal/upload/rewrite_test.go | 2 +- .../internal/upload/saved_file_tracker.go | 13 + .../upload/saved_file_tracker_test.go | 2 +- 171 files changed, 5383 insertions(+), 3962 deletions(-) create mode 100644 app/assets/javascripts/integrations/beyond_identity/components/add_exclusions_drawer.vue create mode 100644 app/assets/javascripts/integrations/beyond_identity/components/exclusions_list.vue create mode 100644 app/assets/javascripts/integrations/beyond_identity/components/exclusions_list_item.vue create mode 100644 app/assets/javascripts/integrations/beyond_identity/components/exclusions_tabs.vue create mode 100644 app/assets/javascripts/integrations/beyond_identity/components/remove_exclusion_confirmation_modal.vue create mode 100644 app/assets/javascripts/vue_shared/components/list_selector/project_item.vue delete mode 100644 app/components/projects/ml/models_index_component.html.haml delete mode 100644 app/components/projects/ml/models_index_component.rb delete mode 100644 app/components/projects/ml/show_ml_model_component.html.haml delete mode 100644 app/components/projects/ml/show_ml_model_component.rb delete mode 100644 app/components/projects/ml/show_ml_model_version_component.html.haml delete mode 100644 app/components/projects/ml/show_ml_model_version_component.rb create mode 100644 config/feature_flags/beta/beyond_identity_exclusions.yml create mode 100644 doc/user/application_security/api_security_testing/checks/application_information_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/authentication_token_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/cleartext_authentication_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/cors_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/dns_rebinding_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/framework_debug_mode_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/heartbleed_open_ssl_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/html_injection_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/index.md create mode 100644 doc/user/application_security/api_security_testing/checks/insecure_http_methods_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/json_hijacking_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/json_injection_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/open_redirect_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/os_command_injection_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/path_traversal_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/sensitive_file_disclosure_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/sensitive_information_disclosure_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/session_cookie_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/shellshock_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/sql_injection_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/tls_server_configuration_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/xml_external_entity_check.md create mode 100644 doc/user/application_security/api_security_testing/checks/xml_injection_check.md create mode 100644 doc/user/application_security/api_security_testing/configuration/customizing_analyzer_settings.md create mode 100644 doc/user/application_security/api_security_testing/configuration/enabling_the_analyzer.md create mode 100644 doc/user/application_security/api_security_testing/configuration/index.md create mode 100644 doc/user/application_security/api_security_testing/configuration/offline_configuration.md create mode 100644 doc/user/application_security/api_security_testing/configuration/overriding_analyzer_jobs.md create mode 100644 doc/user/application_security/api_security_testing/configuration/requirements.md create mode 100644 doc/user/application_security/api_security_testing/configuration/variables.md rename doc/user/application_security/{dast_api => api_security_testing}/img/dast_api_postman_collection_edit_variable.png (100%) rename doc/user/application_security/{dast_api => api_security_testing}/img/dast_api_postman_environment_edit_variable.png (100%) rename doc/user/application_security/{dast_api => api_security_testing}/img/dast_api_postman_request_edit.png (100%) create mode 100644 doc/user/application_security/api_security_testing/index.md create mode 100644 doc/user/application_security/api_security_testing/performance.md create mode 100644 doc/user/application_security/api_security_testing/troubleshooting.md create mode 100644 lib/gitlab/cleanup/orphan_job_artifact_final_objects/paginators/azure.rb delete mode 100644 spec/components/projects/ml/models_index_component_spec.rb delete mode 100644 spec/components/projects/ml/show_ml_model_component_spec.rb delete mode 100644 spec/components/projects/ml/show_ml_model_version_component_spec.rb create mode 100644 spec/frontend/__helpers__/tracking_internal_events_helper.js create mode 100644 spec/frontend/integrations/beyond_identity/components/add_exclusions_drawer_spec.js create mode 100644 spec/frontend/integrations/beyond_identity/components/exclusions_list_item_spec.js create mode 100644 spec/frontend/integrations/beyond_identity/components/exclusions_list_spec.js create mode 100644 spec/frontend/integrations/beyond_identity/components/exclusions_tabs_spec.js create mode 100644 spec/frontend/integrations/beyond_identity/components/mock_data.js create mode 100644 spec/frontend/integrations/beyond_identity/components/remove_exclusion_confirmation_modal_spec.js create mode 100644 spec/frontend/vue_shared/components/list_selector/project_item_spec.js diff --git a/.rubocop_todo/graphql/resolver_method_length.yml b/.rubocop_todo/graphql/resolver_method_length.yml index f27246096a7..f1f50ba4a6f 100644 --- a/.rubocop_todo/graphql/resolver_method_length.yml +++ b/.rubocop_todo/graphql/resolver_method_length.yml @@ -1,6 +1,4 @@ --- GraphQL/ResolverMethodLength: Exclude: - - 'app/graphql/types/ci/detailed_status_type.rb' - - 'app/graphql/types/ci/runner_type.rb' - 'app/graphql/types/ci/stage_type.rb' diff --git a/.rubocop_todo/layout/space_inside_parens.yml b/.rubocop_todo/layout/space_inside_parens.yml index 72db694dc74..4c96a8803ab 100644 --- a/.rubocop_todo/layout/space_inside_parens.yml +++ b/.rubocop_todo/layout/space_inside_parens.yml @@ -129,7 +129,6 @@ Layout/SpaceInsideParens: - 'spec/lib/gitlab/health_checks/simple_check_shared.rb' - 'spec/lib/gitlab/highlight_spec.rb' - 'spec/lib/gitlab/import_export/attributes_permitter_spec.rb' - - 'spec/lib/gitlab/import_export/file_importer_spec.rb' - 'spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb' - 'spec/lib/gitlab/import_export/project/export_task_spec.rb' - 'spec/lib/gitlab/import_export/project/tree_saver_spec.rb' diff --git a/.rubocop_todo/lint/ambiguous_operator_precedence.yml b/.rubocop_todo/lint/ambiguous_operator_precedence.yml index a7cac8214c0..63f21322716 100644 --- a/.rubocop_todo/lint/ambiguous_operator_precedence.yml +++ b/.rubocop_todo/lint/ambiguous_operator_precedence.yml @@ -65,16 +65,6 @@ Lint/AmbiguousOperatorPrecedence: - 'lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb' - 'lib/gitlab/database/postgres_hll/buckets.rb' - 'lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb' - - 'lib/gitlab/i18n/po_linter.rb' - - 'lib/gitlab/import_export/project/relation_factory.rb' - - 'lib/gitlab/memory/instrumentation.rb' - - 'lib/gitlab/metrics/prometheus.rb' - - 'lib/gitlab/middleware/rails_queue_duration.rb' - - 'lib/gitlab/pagination/gitaly_keyset_pager.rb' - - 'lib/gitlab/quick_actions/issuable_actions.rb' - - 'lib/gitlab/relative_positioning.rb' - - 'lib/gitlab/template_parser/parser.rb' - - 'lib/gitlab/tree_summary.rb' - 'lib/omni_auth/strategies/bitbucket.rb' - 'qa/qa/ee/page/project/secure/pipeline_security.rb' - 'qa/qa/resource/api_fabricator.rb' diff --git a/.rubocop_todo/rspec/context_wording.yml b/.rubocop_todo/rspec/context_wording.yml index 758f48cbcbd..c55da3522e5 100644 --- a/.rubocop_todo/rspec/context_wording.yml +++ b/.rubocop_todo/rspec/context_wording.yml @@ -1790,7 +1790,6 @@ RSpec/ContextWording: - 'spec/lib/gitlab/import_export/command_line_util_spec.rb' - 'spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb' - 'spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb' - - 'spec/lib/gitlab/import_export/file_importer_spec.rb' - 'spec/lib/gitlab/import_export/group/object_builder_spec.rb' - 'spec/lib/gitlab/import_export/group/relation_factory_spec.rb' - 'spec/lib/gitlab/import_export/group/tree_restorer_spec.rb' diff --git a/.rubocop_todo/rspec/feature_category.yml b/.rubocop_todo/rspec/feature_category.yml index 545936498bc..df9a3b59ae4 100644 --- a/.rubocop_todo/rspec/feature_category.yml +++ b/.rubocop_todo/rspec/feature_category.yml @@ -3507,7 +3507,6 @@ RSpec/FeatureCategory: - 'spec/lib/gitlab/import_export/design_repo_restorer_spec.rb' - 'spec/lib/gitlab/import_export/design_repo_saver_spec.rb' - 'spec/lib/gitlab/import_export/duration_measuring_spec.rb' - - 'spec/lib/gitlab/import_export/file_importer_spec.rb' - 'spec/lib/gitlab/import_export/group/object_builder_spec.rb' - 'spec/lib/gitlab/import_export/group/relation_factory_spec.rb' - 'spec/lib/gitlab/import_export/group/tree_saver_spec.rb' diff --git a/.rubocop_todo/rspec/named_subject.yml b/.rubocop_todo/rspec/named_subject.yml index ea9f983df4d..5c7be23e996 100644 --- a/.rubocop_todo/rspec/named_subject.yml +++ b/.rubocop_todo/rspec/named_subject.yml @@ -2200,7 +2200,6 @@ RSpec/NamedSubject: - 'spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb' - 'spec/lib/gitlab/import_export/duration_measuring_spec.rb' - 'spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb' - - 'spec/lib/gitlab/import_export/file_importer_spec.rb' - 'spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb' - 'spec/lib/gitlab/import_export/import_failure_service_spec.rb' - 'spec/lib/gitlab/import_export/importer_spec.rb' diff --git a/.rubocop_todo/style/numeric_literal_prefix.yml b/.rubocop_todo/style/numeric_literal_prefix.yml index 3f78dc2e182..f1637ad5640 100644 --- a/.rubocop_todo/style/numeric_literal_prefix.yml +++ b/.rubocop_todo/style/numeric_literal_prefix.yml @@ -47,7 +47,6 @@ Style/NumericLiteralPrefix: - 'spec/lib/gitlab/grape_logging/loggers/cloudflare_logger_spec.rb' - 'spec/lib/gitlab/grape_logging/loggers/queue_duration_logger_spec.rb' - 'spec/lib/gitlab/import_export/command_line_util_spec.rb' - - 'spec/lib/gitlab/import_export/file_importer_spec.rb' - 'spec/lib/gitlab/jwt_authenticatable_spec.rb' - 'spec/lib/gitlab/puma_logging/json_formatter_spec.rb' - 'spec/lib/gitlab/relative_positioning/range_spec.rb' diff --git a/app/assets/javascripts/clusters_list/components/clusters_empty_state.vue b/app/assets/javascripts/clusters_list/components/clusters_empty_state.vue index 339ea3b7c0d..e950613c1fc 100644 --- a/app/assets/javascripts/clusters_list/components/clusters_empty_state.vue +++ b/app/assets/javascripts/clusters_list/components/clusters_empty_state.vue @@ -1,6 +1,7 @@ diff --git a/app/assets/javascripts/integrations/beyond_identity/components/add_exclusions_drawer.vue b/app/assets/javascripts/integrations/beyond_identity/components/add_exclusions_drawer.vue new file mode 100644 index 00000000000..64bbd0e30e6 --- /dev/null +++ b/app/assets/javascripts/integrations/beyond_identity/components/add_exclusions_drawer.vue @@ -0,0 +1,84 @@ + + + diff --git a/app/assets/javascripts/integrations/beyond_identity/components/exclusions_list.vue b/app/assets/javascripts/integrations/beyond_identity/components/exclusions_list.vue new file mode 100644 index 00000000000..89d4695b86f --- /dev/null +++ b/app/assets/javascripts/integrations/beyond_identity/components/exclusions_list.vue @@ -0,0 +1,126 @@ + + + diff --git a/app/assets/javascripts/integrations/beyond_identity/components/exclusions_list_item.vue b/app/assets/javascripts/integrations/beyond_identity/components/exclusions_list_item.vue new file mode 100644 index 00000000000..e44bf92b742 --- /dev/null +++ b/app/assets/javascripts/integrations/beyond_identity/components/exclusions_list_item.vue @@ -0,0 +1,46 @@ + + + diff --git a/app/assets/javascripts/integrations/beyond_identity/components/exclusions_tabs.vue b/app/assets/javascripts/integrations/beyond_identity/components/exclusions_tabs.vue new file mode 100644 index 00000000000..8dd4becee6c --- /dev/null +++ b/app/assets/javascripts/integrations/beyond_identity/components/exclusions_tabs.vue @@ -0,0 +1,39 @@ + + + diff --git a/app/assets/javascripts/integrations/beyond_identity/components/remove_exclusion_confirmation_modal.vue b/app/assets/javascripts/integrations/beyond_identity/components/remove_exclusion_confirmation_modal.vue new file mode 100644 index 00000000000..6ba61e97b34 --- /dev/null +++ b/app/assets/javascripts/integrations/beyond_identity/components/remove_exclusion_confirmation_modal.vue @@ -0,0 +1,58 @@ + + diff --git a/app/assets/javascripts/pages/admin/integrations/overrides/index.js b/app/assets/javascripts/pages/admin/integrations/overrides/index.js index b1504709144..437c77dd25c 100644 --- a/app/assets/javascripts/pages/admin/integrations/overrides/index.js +++ b/app/assets/javascripts/pages/admin/integrations/overrides/index.js @@ -1,3 +1,33 @@ +import Vue from 'vue'; +import VueApollo from 'vue-apollo'; +import createDefaultClient from '~/lib/graphql'; import initIntegrationOverrides from '~/integrations/overrides'; +import ExclusionsList from '~/integrations/beyond_identity/components/exclusions_list.vue'; + +const initBeyondIdentityExclusions = () => { + const el = document.querySelector('.js-vue-beyond-identity-exclusions'); + + if (!el) { + return null; + } + + const { editPath } = el.dataset; + + return new Vue({ + el, + + apolloProvider: new VueApollo({ + defaultClient: createDefaultClient(), + }), + provide: { + editPath, + }, + render(createElement) { + return createElement(ExclusionsList); + }, + }); +}; + +initBeyondIdentityExclusions(); initIntegrationOverrides(); diff --git a/app/assets/javascripts/tracking/internal_events.js b/app/assets/javascripts/tracking/internal_events.js index 1818136a684..099c6db843d 100644 --- a/app/assets/javascripts/tracking/internal_events.js +++ b/app/assets/javascripts/tracking/internal_events.js @@ -9,6 +9,8 @@ import { validateAdditionalProperties, } from './utils'; +const elementsWithBinding = new WeakMap(); + const InternalEvents = { /** * @@ -52,23 +54,30 @@ const InternalEvents = { /** * Attaches event handlers for data-attributes powered events. * - * @param {HTMLElement} parent - element containing data-attributes - * @returns {Object} handler - object containing name of the event and its corresponding function + * @param {HTMLElement} parent - element containing data-attributes to which the event listener + * will be attached. + * @returns {Function|null} A dispose function that can be called to remove the event listener and + * unmark the element, or null if no event handler was attached. */ bindInternalEventDocument(parent = document) { - if (!Tracker.enabled() || parent.internalEventsTrackingBound) { - return []; + if (!Tracker.enabled() || elementsWithBinding.has(parent)) { + return null; } - // eslint-disable-next-line no-param-reassign - parent.internalEventsTrackingBound = true; + elementsWithBinding.set(parent, true); - const handler = { - name: 'click', - func: (e) => InternalEventHandler(e, this.trackEvent.bind(this)), + const eventName = 'click'; + const eventFunc = (e) => InternalEventHandler(e, this.trackEvent.bind(this)); + + parent.addEventListener(eventName, eventFunc); + + const dispose = () => { + elementsWithBinding.delete(parent); + + parent.removeEventListener(eventName, eventFunc); }; - parent.addEventListener(handler.name, handler.func); - return handler; + + return dispose; }, /** * Attaches internal event handlers for load events. diff --git a/app/assets/javascripts/vue_shared/components/incubation/incubation_alert.vue b/app/assets/javascripts/vue_shared/components/incubation/incubation_alert.vue index b704cec2475..017502240cb 100644 --- a/app/assets/javascripts/vue_shared/components/incubation/incubation_alert.vue +++ b/app/assets/javascripts/vue_shared/components/incubation/incubation_alert.vue @@ -1,6 +1,7 @@ @@ -54,7 +56,7 @@ export default { @dismiss="dismissAlert" > {{ $options.i18n.contentLabel }} - {{ + {{ $options.i18n.learnMoreLabel }} diff --git a/app/assets/javascripts/vue_shared/components/list_selector/constants.js b/app/assets/javascripts/vue_shared/components/list_selector/constants.js index ad826c6f3e5..5ff5d59ef6b 100644 --- a/app/assets/javascripts/vue_shared/components/list_selector/constants.js +++ b/app/assets/javascripts/vue_shared/components/list_selector/constants.js @@ -2,6 +2,7 @@ import { __ } from '~/locale'; import UserItem from './user_item.vue'; import GroupItem from './group_item.vue'; import DeployKeyItem from './deploy_key_item.vue'; +import ProjectItem from './project_item.vue'; export const CONFIG = { users: { @@ -23,4 +24,10 @@ export const CONFIG = { filterKey: 'name', component: DeployKeyItem, }, + projects: { + title: __('Projects'), + icon: 'project', + filterKey: 'id', + component: ProjectItem, + }, }; diff --git a/app/assets/javascripts/vue_shared/components/list_selector/index.vue b/app/assets/javascripts/vue_shared/components/list_selector/index.vue index d79a8d6a00c..67b2e341ce0 100644 --- a/app/assets/javascripts/vue_shared/components/list_selector/index.vue +++ b/app/assets/javascripts/vue_shared/components/list_selector/index.vue @@ -1,10 +1,11 @@ + + diff --git a/app/assets/javascripts/work_items/components/shared/work_item_link_child_contents.vue b/app/assets/javascripts/work_items/components/shared/work_item_link_child_contents.vue index 9aa3f1e24af..925ff2376d5 100644 --- a/app/assets/javascripts/work_items/components/shared/work_item_link_child_contents.vue +++ b/app/assets/javascripts/work_items/components/shared/work_item_link_child_contents.vue @@ -202,7 +202,6 @@ export default { data-testid="item-status-icon" > { with_status(:failed, :canceled, :canceling) } scope :complete, -> { with_status(completed_statuses) } scope :incomplete, -> { without_statuses(completed_statuses) } + scope :complete_or_manual, -> { with_status(completed_with_manual_statuses) } scope :waiting_for_resource_or_upcoming, -> { with_status(:created, :scheduled, :waiting_for_resource) } scope :cancelable, -> do diff --git a/app/models/members/members/invited_private_group_accessibility_assigner.rb b/app/models/members/members/invited_private_group_accessibility_assigner.rb index 8ac1600f6e5..e4dafd56a16 100644 --- a/app/models/members/members/invited_private_group_accessibility_assigner.rb +++ b/app/models/members/members/invited_private_group_accessibility_assigner.rb @@ -45,7 +45,7 @@ module Members return [] if current_user.nil? private_invited_groups = private_invited_group_members.map(&:source).uniq - private_invited_groups.select { |group| current_user.can?(:read_group, group) } + Group.groups_user_can(private_invited_groups, current_user, :read_group) end strong_memoize_attr(:authorized_groups) diff --git a/app/views/shared/integrations/_tabs.html.haml b/app/views/shared/integrations/_tabs.html.haml index 781db59592e..bb94590bb98 100644 --- a/app/views/shared/integrations/_tabs.html.haml +++ b/app/views/shared/integrations/_tabs.html.haml @@ -1,9 +1,10 @@ +- custom_settings_title = @integration.title == 'Beyond Identity' && Feature.enabled?(:beyond_identity_exclusions) ? s_('Integrations|Exclusions') : s_('Integrations|Projects using custom settings') - if integration.instance_level? .tabs.gl-tabs %div = gl_tabs_nav({ class: 'gl-mb-5' }) do = gl_tab_link_to _('Settings'), scoped_edit_integration_path(integration, project: @project, group: @group) - = gl_tab_link_to s_('Integrations|Projects using custom settings'), scoped_overrides_integration_path(integration) + = gl_tab_link_to custom_settings_title, scoped_overrides_integration_path(integration) = yield diff --git a/app/views/shared/integrations/overrides.html.haml b/app/views/shared/integrations/overrides.html.haml index c25527a605c..5d67f63bb2a 100644 --- a/app/views/shared/integrations/overrides.html.haml +++ b/app/views/shared/integrations/overrides.html.haml @@ -5,4 +5,8 @@ %h1.page-title.gl-font-size-h-display = @integration.title -.js-vue-integration-overrides{ data: integration_overrides_data(@integration, project: @project, group: @group) } +- if @integration.title == 'Beyond Identity' && Feature.enabled?(:beyond_identity_exclusions) + .js-vue-beyond-identity-exclusions{ data: integration_overrides_data(@integration) } + +- else + .js-vue-integration-overrides{ data: integration_overrides_data(@integration, project: @project, group: @group) } diff --git a/app/workers/projects/import_export/relation_import_worker.rb b/app/workers/projects/import_export/relation_import_worker.rb index c30b20593c4..9b3147c7832 100644 --- a/app/workers/projects/import_export/relation_import_worker.rb +++ b/app/workers/projects/import_export/relation_import_worker.rb @@ -40,6 +40,8 @@ module Projects tracker.fail_op! raise + ensure + remove_extracted_import end private @@ -47,20 +49,29 @@ module Projects def extract_import_file Gitlab::ImportExport::FileImporter.import( importable: project, - archive_file: project.import_export_upload.import_file.path, - shared: shared_export_data + archive_file: nil, + shared: project.import_export_shared, + tmpdir: tmpdir ) end + def remove_extracted_import + FileUtils.rm_rf(tmpdir) + end + + def tmpdir + @tmpdir ||= Dir.mktmpdir('export_archives') + end + def process_import tree_restorer = Gitlab::ImportExport::Project::RelationTreeRestorer.new( user: current_user, - shared: shared_export_data, + shared: project.import_export_shared, relation_reader: relation_reader, object_builder: Gitlab::ImportExport::Project::ObjectBuilder, members_mapper: members_mapper, relation_factory: Gitlab::ImportExport::Project::RelationFactory, - reader: Gitlab::ImportExport::Reader.new(shared: shared_export_data), + reader: Gitlab::ImportExport::Reader.new(shared: project.import_export_shared), importable: project, importable_attributes: relation_reader.consume_attributes('project'), importable_path: 'project', @@ -72,7 +83,7 @@ module Projects def relation_reader @relation_reader ||= Gitlab::ImportExport::Json::NdjsonReader.new( - File.join(shared_export_data.export_path, 'tree') + File.join(tmpdir, 'tree') ) end @@ -87,10 +98,6 @@ module Projects importable: project ) end - - def shared_export_data - @shared ||= project.import_export_shared - end end end end diff --git a/config/events/20230321151607_github_create.yml b/config/events/20230321151607_github_create.yml index 7dc42334cfe..b64003323b6 100644 --- a/config/events/20230321151607_github_create.yml +++ b/config/events/20230321151607_github_create.yml @@ -16,7 +16,7 @@ identifiers: - project product_section: dev product_stage: manage -product_group: import +product_group: import_and_integrate milestone: "15.10" introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112062 distributions: diff --git a/config/events/20230407142541_Gitlab__GithubGistsImport__ImportGistWorker_create.yml b/config/events/20230407142541_Gitlab__GithubGistsImport__ImportGistWorker_create.yml index 59af78f00bf..4dcd9beb509 100644 --- a/config/events/20230407142541_Gitlab__GithubGistsImport__ImportGistWorker_create.yml +++ b/config/events/20230407142541_Gitlab__GithubGistsImport__ImportGistWorker_create.yml @@ -13,7 +13,7 @@ identifiers: - user product_section: dev product_stage: manage -product_group: import +product_group: import_and_integrate milestone: "15.11" introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117057 distributions: diff --git a/config/feature_flags/beta/beyond_identity_exclusions.yml b/config/feature_flags/beta/beyond_identity_exclusions.yml new file mode 100644 index 00000000000..e599a4a93c1 --- /dev/null +++ b/config/feature_flags/beta/beyond_identity_exclusions.yml @@ -0,0 +1,9 @@ +--- +name: beyond_identity_exclusions +feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/454372 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/150664 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/457893 +milestone: '17.0' +group: group::source code +type: beta +default_enabled: false diff --git a/data/deprecations/15-9-deprecate-legacy-requirements-iid.yml b/data/deprecations/15-9-deprecate-legacy-requirements-iid.yml index 05ae0feaa47..eb1bdad8bbe 100644 --- a/data/deprecations/15-9-deprecate-legacy-requirements-iid.yml +++ b/data/deprecations/15-9-deprecate-legacy-requirements-iid.yml @@ -1,10 +1,10 @@ - title: 'The GitLab legacy requirement IID is deprecated in favor of work item IID' announcement_milestone: '15.9' - removal_milestone: '17.0' + removal_milestone: '18.0' breaking_change: true reporter: mmacfarlane body: | - We will be transitioning to a new IID as a result of moving requirements to a [work item type](https://docs.gitlab.com/ee/development/work_items.html#work-items-and-work-item-types). Users should begin using the new IID as support for the legacy IID and existing formatting will end in GitLab 17.0. The legacy requirement IID remains available until its removal in GitLab 17.0. + We will be transitioning to a new IID as a result of moving requirements to a [work item type](https://docs.gitlab.com/ee/development/work_items.html#work-items-and-work-item-types). Users should begin using the new IID as support for the legacy IID and existing formatting will end in GitLab 18.0. The legacy requirement IID remains available until its removal in GitLab 18.0. stage: plan issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/390263 diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index d5c7b737e40..2e9337713a4 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -18691,6 +18691,7 @@ Represents a product analytics dashboard. | `errors` | [`[String!]`](#string) | Errors on yaml definition. | | `panels` | [`CustomizableDashboardPanelConnection!`](#customizabledashboardpanelconnection) | Panels shown on the dashboard. (see [Connections](#connections)) | | `slug` | [`String!`](#string) | Slug of the dashboard. | +| `status` **{warning-solid}** | [`String`](#string) | **Introduced** in GitLab 17.0. **Status**: Experiment. Status of the dashboard. | | `title` | [`String!`](#string) | Title of the dashboard. | | `userDefined` | [`Boolean!`](#boolean) | Indicates whether the dashboard is user-defined or provided by GitLab. | diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index 4b5da8f09fc..cb2e767a036 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -1435,7 +1435,7 @@ job: #### `artifacts:access` -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/145206) in GitLab 16.10. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/145206) in GitLab 16.11. Use `artifacts:access` to determine who can access the job artifacts. diff --git a/doc/development/internal_analytics/internal_event_instrumentation/quick_start.md b/doc/development/internal_analytics/internal_event_instrumentation/quick_start.md index 9aa95a7c24c..92a197ca8de 100644 --- a/doc/development/internal_analytics/internal_event_instrumentation/quick_start.md +++ b/doc/development/internal_analytics/internal_event_instrumentation/quick_start.md @@ -269,3 +269,148 @@ For Haml: ```haml = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle', data: { event_tracking: 'action', event_label: 'group_runner_form', event_property: dynamic_property_var, event_value: 2 }}) do ``` + +#### Frontend testing + +If you are using the `trackEvent` method in any of your code, whether it is in raw JavaScript or a Vue component, you can use the `useMockInternalEventsTracking` helper method to assert if `trackEvent` is called. + +For example, if we need to test the below Vue component, + +```vue + + +``` + +Below would be the test case for above component. + +```javascript +import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; +import DeleteApplication from '~/admin/applications/components/delete_application.vue'; +import { useMockInternalEventsTracking } from 'helpers/tracking_internal_events_helper'; + +describe('DeleteApplication', () => { + /** @type {import('helpers/vue_test_utils_helper').ExtendedWrapper} */ + let wrapper; + + const createComponent = () => { + wrapper = shallowMountExtended(DeleteApplication); + }; + + beforeEach(() => { + createComponent(); + }); + + describe('sample button 1', () => { + const { bindInternalEventDocument } = useMockInternalEventsTracking(); + it('should call trackEvent method when clicked on sample button', async () => { + const { trackEventSpy } = bindInternalEventDocument(wrapper.element); + + await wrapper.find('.sample-button').vm.$emit('click'); + + expect(trackEventSpy).toHaveBeenCalledWith( + 'click_view_runners_button', + { + label: 'group_runner_form', + property: 'property_value', + value: 3, + }, + undefined, + ); + }); + }); +}); +``` + +If you are using tracking attributes for in Vue/View templates like below, + +```vue + + +``` + +Below would be the test case for above component. + +```javascript +import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; +import DeleteApplication from '~/admin/applications/components/delete_application.vue'; +import { useMockInternalEventsTracking } from 'helpers/tracking_internal_events_helper'; + +describe('DeleteApplication', () => { + /** @type {import('helpers/vue_test_utils_helper').ExtendedWrapper} */ + let wrapper; + + const createComponent = () => { + wrapper = shallowMountExtended(DeleteApplication); + }; + + beforeEach(() => { + createComponent(); + }); + + describe('sample button', () => { + const { bindInternalEventDocument } = useMockInternalEventsTracking(); + it('should call trackEvent method when clicked on sample button', () => { + const { triggerEvent, trackEventSpy } = bindInternalEventDocument(wrapper.element); + triggerEvent('.sample-button'); + expect(trackEventSpy).toHaveBeenCalledWith('click_view_runners_button', { + label: 'group_runner_form', + }); + }); + }); +}); +``` diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md index 0f06116b772..ec5bd8380b1 100644 --- a/doc/update/deprecations.md +++ b/doc/update/deprecations.md @@ -455,6 +455,20 @@ From GitLab 18.0 and later, the runner registration methods implemented by the n
+### The GitLab legacy requirement IID is deprecated in favor of work item IID + +
+- Announced in GitLab 15.9 +- Removal in GitLab 18.0 ([breaking change](https://docs.gitlab.com/ee/update/terminology.html#breaking-change)) +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/390263). +
+ +We will be transitioning to a new IID as a result of moving requirements to a [work item type](https://docs.gitlab.com/ee/development/work_items.html#work-items-and-work-item-types). Users should begin using the new IID as support for the legacy IID and existing formatting will end in GitLab 18.0. The legacy requirement IID remains available until its removal in GitLab 18.0. + +
+ +
+ ### The `Project.services` GraphQL field is deprecated
@@ -1997,20 +2011,6 @@ Instead, GitHub repositories can be imported by using the [API](https://docs.git
-### The GitLab legacy requirement IID is deprecated in favor of work item IID - -
-- Announced in GitLab 15.9 -- Removal in GitLab 17.0 ([breaking change](https://docs.gitlab.com/ee/update/terminology.html#breaking-change)) -- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/390263). -
- -We will be transitioning to a new IID as a result of moving requirements to a [work item type](https://docs.gitlab.com/ee/development/work_items.html#work-items-and-work-item-types). Users should begin using the new IID as support for the legacy IID and existing formatting will end in GitLab 17.0. The legacy requirement IID remains available until its removal in GitLab 17.0. - -
- -
- ### The Visual Reviews tool is deprecated
diff --git a/doc/user/application_security/api_security_testing/checks/application_information_check.md b/doc/user/application_security/api_security_testing/checks/application_information_check.md new file mode 100644 index 00000000000..27d95437e43 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/application_information_check.md @@ -0,0 +1,24 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Application information disclosure + +## Description + +Application information disclosure check. This includes information such as version numbers, database error messages, stack traces. + +## Remediation + +Application information disclosure is an application weakness where an application reveals sensitive data, such as technical details of the web application or environment. Application data may be used by an attacker to exploit the target web application, its hosting network, or its users. Therefore, leakage of sensitive data should be limited or prevented whenever possible. Information disclosure, in its most common form, is the result of one or more of the following conditions: a failure to scrub out HTML or script comments containing sensitive information or improper application or server configurations. + +Failure to scrub HTML or script comments prior to a push to the production environment can result in the leak of sensitive, contextual, information such as server directory structure, SQL query structure, and internal network information. Often a developer will leave comments within the HTML and script code to help facilitate the debugging or integration process during the pre-production phase. Although there is no harm in allowing developers to include inline comments within the content they develop, these comments should all be removed prior to the content's public release. + +Software version numbers and verbose error messages (such as ASP.NET version numbers) are examples of improper server configurations. This information is useful to an attacker by providing detailed insight as to the framework, languages, or pre-built functions being utilized by a web application. Most default server configurations provide software version numbers and verbose error messages for debugging and troubleshooting purposes. Configuration changes can be made to disable these features, preventing the display of this information. + +## Links + +- [OWASP](https://owasp.org/Top10/A05_2021-Security_Misconfiguration) +- [CWE](https://cwe.mitre.org/data/definitions/200.html) diff --git a/doc/user/application_security/api_security_testing/checks/authentication_token_check.md b/doc/user/application_security/api_security_testing/checks/authentication_token_check.md new file mode 100644 index 00000000000..1bd5e8fdb61 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/authentication_token_check.md @@ -0,0 +1,29 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Authentication token + +## Description + +Perform various authentication token checks such as removing the token or changing to an invalid value. + +## Remediation + +API tokens must be unpredictable (random enough) to prevent guessing attacks, where an attacker is able to guess or predict a valid API Token through statistical analysis techniques. For this purpose, a good PRNG (Pseudo Random Number Generator) must be used. + +The authentication token may have been: + +- modified to an invalid value. +- removed from request. +- not match length requirements. +- configured as a signature. + +An API operation failed to property restrict access using an authentication token. This allows an attacker to bypass authentication gaining access to information or even the ability to modify data. + +## Links + +- [OWASP](https://owasp.org/Top10/A07_2021-Identification_and_Authentication_Failures) +- [CWE](https://cwe.mitre.org/data/definitions/285.html) diff --git a/doc/user/application_security/api_security_testing/checks/cleartext_authentication_check.md b/doc/user/application_security/api_security_testing/checks/cleartext_authentication_check.md new file mode 100644 index 00000000000..e8a6ae4e4e5 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/cleartext_authentication_check.md @@ -0,0 +1,20 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Cleartext authentication + +## Description + +This check looks for cleartext authentication such as HTTP Basic auth with no-TLS. + +## Remediation + +Authentication credentials are transported via unencrypted channel (HTTP). This exposes the transmitted credentials to any attacker who can monitor (sniff) the network traffic during transmission. Sensitive information such as credentials should always be transmitted via encrypted channels such as HTTPS. + +## Links + +- [OWASP](https://owasp.org/Top10/A02_2021-Cryptographic_Failures) +- [CWE](https://cwe.mitre.org/data/definitions/319.html) diff --git a/doc/user/application_security/api_security_testing/checks/cors_check.md b/doc/user/application_security/api_security_testing/checks/cors_check.md new file mode 100644 index 00000000000..eb22d0a8211 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/cors_check.md @@ -0,0 +1,20 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# CORS + +## Description + +Check for CORS misconfiguration including overly permissive white-lists of accepted Origin headers or failure to validate Origin header. Also checks for allowing credentials on potentially invalid or dangerous Origins and missing headers that could potentially result in cache poisoning. + +## Remediation + +A misconfigured CORS implementation may be overly permissive in which domains should be trusted and at what level of trust. This could allow an untrusted domain to forge the Origin header and launch various types of attacks such as cross-site request forgery or cross-site scripting. An attacker could potentially steal a victim's credentials or send malicious requests on behalf of a victim. The victim may not even be aware that an attack is being launched. + +## Links + +- [OWASP](https://owasp.org/Top10/A01_2021-Broken_Access_Control) +- [CWE](https://cwe.mitre.org/data/definitions/942.html) diff --git a/doc/user/application_security/api_security_testing/checks/dns_rebinding_check.md b/doc/user/application_security/api_security_testing/checks/dns_rebinding_check.md new file mode 100644 index 00000000000..ca80746928c --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/dns_rebinding_check.md @@ -0,0 +1,20 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# DNS rebinding + +## Description + +Check for DNS rebinding. This check verifies that the host checks that the HOST header of the request exists and matches the expected name of the host to avoid attacks via malicious DNS entries. + +## Remediation + +DNS rebinding allows a malicious host to spoof or redirect a request to an alternate IP address, potentially allowing an attacker to bypass security authentication or authorization. DNS resolution on its own does not properly constitute a valid authentication mechanism. Servers should validate that the Host header of the request matches the expected hostname of the server. In cases where the hostname is missing or does not match the expected value, the server should return a 400. The X-Forwarded-Host header is sometimes used instead of the Host header in cases where the request is being forwarded. In these cases, the X-Forwarded-Host header should also be validated if it is being used to determine the Host of the original request. + +## Links + +- [OWASP](https://owasp.org/Top10/A05_2021-Security_Misconfiguration) +- [CWE](https://cwe.mitre.org/data/definitions/350.html) diff --git a/doc/user/application_security/api_security_testing/checks/framework_debug_mode_check.md b/doc/user/application_security/api_security_testing/checks/framework_debug_mode_check.md new file mode 100644 index 00000000000..4e8ef50d859 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/framework_debug_mode_check.md @@ -0,0 +1,21 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Framework debug mode + +## Description + +Checks to see if debug mode is enabled in various frameworks such as Flask and ASP.NET. This check has a low false positive rate. + +## Remediation + +The Flask or ASP .NET framework was identified with debug mode enabled. This allows an attacker the ability to download any file on the file system and other capabilities. This is a high severity issue that is easy for an attacker to exploit. + +## Links + +- [OWASP](https://owasp.org/Top10/A05_2021-Security_Misconfiguration) +- [CWE-23: Relative Path Traversal](https://cwe.mitre.org/data/definitions/23.html) +- [CWE-285: Improper Authorization](https://cwe.mitre.org/data/definitions/285.html) diff --git a/doc/user/application_security/api_security_testing/checks/heartbleed_open_ssl_check.md b/doc/user/application_security/api_security_testing/checks/heartbleed_open_ssl_check.md new file mode 100644 index 00000000000..b6abc3691b4 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/heartbleed_open_ssl_check.md @@ -0,0 +1,22 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Heartbleed OpenSSL vulnerability + +## Description + +Check for Heartbleed OpenSSL vulnerability. + +## Remediation + +The Heartbleed vulnerability is a serious bug in the popular OpenSSL cryptographic library. OpenSSL is used to encrypt and decrypt communications and secure the Internet traffic. This vulnerability allows the attacker to steal protected information, which should not be accessible under other circumstance such as secret keys that are used to encrypt sensitive information. + +Anyone on with access to the target API can use the Heartbleed vulnerability to read the memory from protected systems taking advantage of vulnerable versions of OpenSSL library. + +## Links + +- [OWASP](https://owasp.org/Top10/A06_2021-Vulnerable_and_Outdated_Components) +- [CWE](https://cwe.mitre.org/data/definitions/119.html) diff --git a/doc/user/application_security/api_security_testing/checks/html_injection_check.md b/doc/user/application_security/api_security_testing/checks/html_injection_check.md new file mode 100644 index 00000000000..ab273bbae2b --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/html_injection_check.md @@ -0,0 +1,28 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# HTML injection + +## Description + +Check for XSS via HTML injection into all fields that support strings. This includes portions of the HTTP request such as path, query, headers and also body parameters such as XML fields, JSON fields, etc. Detection is performed by monitoring responses for the injected value in to known HTML enabled fields. + +## Remediation + +Cross-site scripting (XSS) is an attack technique that involves echoing attacker-supplied code into a user's browser instance. A browser instance can be a standard web browser client, or a browser object embedded in a software product such as the browser within WinAmp, an RSS reader, or an email client. The code itself is usually written in HTML/JavaScript, but may also extend to VBScript, ActiveX, Java, Flash, or any other browser-supported technology. + +When an attacker gets a user's browser to execute his/her code, the code will run within the security context (or zone) of the hosting web site. With this level of privilege, the code has the ability to read, modify and transmit any sensitive data accessible by the browser. A Cross-site Scripted user could have his/her account hijacked (cookie theft), their browser redirected to another location, or possibly shown fraudulent content delivered by the web site they are visiting. Cross-site Scripting attacks essentially compromise the trust relationship between a user and the web site. Applications utilizing browser object instances which load content from the file system may execute code under the local machine zone allowing for system compromise. + +There are three types of Cross-site Scripting attacks: non-persistent, persistent and DOM-based. + +Non-persistent attacks and DOM-based attacks require a user to either visit a specially crafted link laced with malicious code, or visit a malicious web page containing a web form, which when posted to the vulnerable site, will mount the attack. Using a malicious form will oftentimes take place when the vulnerable resource only accepts HTTP POST requests. In such a case, the form can be submitted automatically, without the victim's knowledge (e.g. by using JavaScript). Upon clicking on the malicious link or submitting the malicious form, the XSS payload will get echoed back and will get interpreted by the user's browser and execute. Another technique to send almost arbitrary requests (GET and POST) is by using an embedded client, such as Adobe Flash. + +Persistent attacks occur when the malicious code is submitted to a web site where it's stored for a period of time. Examples of an attacker's favorite targets often include message board posts, web mail messages, and web chat software. The unsuspecting user is not required to interact with any additional site/link (e.g. an attacker site or a malicious link sent via email), just simply view the web page containing the code. + +## Links + +- [OWASP](https://owasp.org/Top10/A03_2021-Injection) +- [CWE](https://cwe.mitre.org/data/definitions/79.html) diff --git a/doc/user/application_security/api_security_testing/checks/index.md b/doc/user/application_security/api_security_testing/checks/index.md new file mode 100644 index 00000000000..a46a9e79042 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/index.md @@ -0,0 +1,140 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# API security testing vulnerability checks + +DETAILS: +**Tier:** Ultimate +**Offering:** GitLab.com, Self-managed, GitLab Dedicated + +> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/457449) from **DAST API vulnerability checks** to **API security testing vulnerability checks** in GitLab 17.0. + +[API security testing](../index.md) provides vulnerability checks that are used to +scan for vulnerabilities in the API under test. + +## Passive checks + +| Check | Severity | Type | Profiles | +|:-----------------------------------------------------------------------------|:---------|:--------|:---------| +| [Application information check](application_information_check.md) | Medium | Passive | Passive, Passive-Quick, Active-Quick, Active Full, Quick, Full | +| [Cleartext authentication check](cleartext_authentication_check.md) | High | Passive | Passive, Passive-Quick, Active-Quick, Active Full, Quick, Full | +| [JSON hijacking](json_hijacking_check.md) | Medium | Passive | Passive, Passive-Quick, Active-Quick, Active Full, Quick, Full | +| [Sensitive information](sensitive_information_disclosure_check.md) | High | Passive | Passive, Passive-Quick, Active-Quick, Active Full, Quick, Full | +| [Session cookie](session_cookie_check.md) | Medium | Passive | Passive, Passive-Quick, Active-Quick, Active Full, Quick, Full | + +## Active checks + +| Check | Severity | Type | Profiles | +|:-----------------------------------------------------------------------------|:---------|:--------|:---------| +| [CORS](cors_check.md) | Medium | Active | Active Full, Full | +| [DNS rebinding](dns_rebinding_check.md) | Medium | Active | Active Full, Full | +| [Framework debug mode](framework_debug_mode_check.md) | High | Active | Active-Quick, Active Full, Quick, Full | +| [Heartbleed OpenSSL vulnerability](heartbleed_open_ssl_check.md) | High | Active | Active Full, Full | +| [HTML injection check](html_injection_check.md) | Medium | Active | Active-Quick, Active Full, Quick, Full | +| [Insecure HTTP methods](insecure_http_methods_check.md) | Medium | Active | Active-Quick, Active Full, Quick, Full | +| [JSON injection](json_injection_check.md) | Medium | Active | Active-Quick, Active Full, Quick, Full | +| [Open redirect](open_redirect_check.md) | Medium | Active | Active Full, Full | +| [OS command injection](os_command_injection_check.md) | High | Active | Active-Quick, Active Full, Quick, Full | +| [Path traversal](path_traversal_check.md) | High | Active | Active Full, Full | +| [Sensitive file](sensitive_file_disclosure_check.md) | Medium | Active | Active Full, Full | +| [Shellshock](shellshock_check.md) | High | Active | Active Full, Full | +| [SQL injection](sql_injection_check.md) | High | Active | Active-Quick, Active Full, Quick, Full | +| [TLS configuration](tls_server_configuration_check.md) | High | Active | Active Full, Full | +| [Authentication token](authentication_token_check.md) | High | Active | Active-Quick, Active Full, Quick, Full | +| [XML external entity](xml_external_entity_check.md) | High | Active | Active Full, Full | +| [XML injection](xml_injection_check.md) | Medium | Active | Active-Quick, Active Full, Quick, Full | + +## API security testing checks by profile + +### Passive-Quick + +- [Application information check](application_information_check.md) +- [Cleartext authentication check](cleartext_authentication_check.md) +- [JSON hijacking](json_hijacking_check.md) +- [Sensitive information](sensitive_information_disclosure_check.md) +- [Session cookie](session_cookie_check.md) + +### Active-Quick + +- [Application information check](application_information_check.md) +- [Cleartext authentication check](cleartext_authentication_check.md) +- [Framework debug mode](framework_debug_mode_check.md) +- [HTML injection check](html_injection_check.md) +- [Insecure HTTP methods](insecure_http_methods_check.md) +- [JSON hijacking](json_hijacking_check.md) +- [JSON injection](json_injection_check.md) +- [OS command injection](os_command_injection_check.md) +- [Sensitive information](sensitive_information_disclosure_check.md) +- [Session cookie](session_cookie_check.md) +- [SQL injection](sql_injection_check.md) +- [Authentication token](authentication_token_check.md) +- [XML injection](xml_injection_check.md) + +### Active-Full + +- [Application information check](application_information_check.md) +- [Cleartext authentication check](cleartext_authentication_check.md) +- [CORS](cors_check.md) +- [DNS rebinding](dns_rebinding_check.md) +- [Framework debug mode](framework_debug_mode_check.md) +- [Heartbleed OpenSSL vulnerability](heartbleed_open_ssl_check.md) +- [HTML injection check](html_injection_check.md) +- [Insecure HTTP methods](insecure_http_methods_check.md) +- [JSON hijacking](json_hijacking_check.md) +- [JSON injection](json_injection_check.md) +- [Open redirect](open_redirect_check.md) +- [OS command injection](os_command_injection_check.md) +- [Path traversal](path_traversal_check.md) +- [Sensitive file](sensitive_file_disclosure_check.md) +- [Sensitive information](sensitive_information_disclosure_check.md) +- [Session cookie](session_cookie_check.md) +- [Shellshock](shellshock_check.md) +- [SQL injection](sql_injection_check.md) +- [TLS configuration](tls_server_configuration_check.md) +- [Authentication token](authentication_token_check.md) +- [XML injection](xml_injection_check.md) +- [XML external entity](xml_external_entity_check.md) + +### Quick + +- [Application information check](application_information_check.md) +- [Cleartext authentication check](cleartext_authentication_check.md) +- [Framework debug mode](framework_debug_mode_check.md) +- [HTML injection check](html_injection_check.md) +- [Insecure HTTP methods](insecure_http_methods_check.md) +- [JSON hijacking](json_hijacking_check.md) +- [JSON injection](json_injection_check.md) +- [OS command injection](os_command_injection_check.md) +- [Sensitive information](sensitive_information_disclosure_check.md) +- [Session cookie](session_cookie_check.md) +- [SQL injection](sql_injection_check.md) +- [Authentication token](authentication_token_check.md) +- [XML injection](xml_injection_check.md) + +### Full + +- [Application information check](application_information_check.md) +- [Cleartext authentication check](cleartext_authentication_check.md) +- [CORS](cors_check.md) +- [DNS rebinding](dns_rebinding_check.md) +- [Framework debug mode](framework_debug_mode_check.md) +- [Heartbleed OpenSSL vulnerability](heartbleed_open_ssl_check.md) +- [HTML injection check](html_injection_check.md) +- [Insecure HTTP methods](insecure_http_methods_check.md) +- [JSON hijacking](json_hijacking_check.md) +- [JSON injection](json_injection_check.md) +- [Open redirect](open_redirect_check.md) +- [OS command injection](os_command_injection_check.md) +- [Path traversal](path_traversal_check.md) +- [Sensitive file](sensitive_file_disclosure_check.md) +- [Sensitive information](sensitive_information_disclosure_check.md) +- [Session cookie](session_cookie_check.md) +- [Shellshock](shellshock_check.md) +- [SQL injection](sql_injection_check.md) +- [TLS configuration](tls_server_configuration_check.md) +- [Authentication token](authentication_token_check.md) +- [XML injection](xml_injection_check.md) +- [XML external entity](xml_external_entity_check.md) diff --git a/doc/user/application_security/api_security_testing/checks/insecure_http_methods_check.md b/doc/user/application_security/api_security_testing/checks/insecure_http_methods_check.md new file mode 100644 index 00000000000..aabcef34752 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/insecure_http_methods_check.md @@ -0,0 +1,22 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Insecure HTTP methods + +## Description + +Checks to see if HTTP methods like OPTIONS and TRACE are enabled on any target endpoints. + +## Remediation + +The resource tested supports the OPTIONS HTTP method. Normally this is considered a security miss configuration as it leaks supported HTTP methods leading to information gathering about a specific server or resource. However, there is a sub-set of the API community looking to use OPTIONS as a method to self discover resource operations. If this is the intended use for enabling OPTIONS, than this issue can be considered a false positive. + +The resource tested supports the TRACE HTTP method. In combination with other cross-domain vulnerabilities in web browsers, sensitive information can be leaked from headers. It's recommended the TRACE method be disabled in your server/framework. + +## Links + +- [OWASP](https://owasp.org/Top10/A05_2021-Security_Misconfiguration) +- [CWE](https://cwe.mitre.org/data/definitions/200.html) diff --git a/doc/user/application_security/api_security_testing/checks/json_hijacking_check.md b/doc/user/application_security/api_security_testing/checks/json_hijacking_check.md new file mode 100644 index 00000000000..af07dd83ecb --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/json_hijacking_check.md @@ -0,0 +1,20 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# JSON hijacking + +## Description + +Checks for JSON data potentially vulnerable to hijacking. This check looks for a GET request that returns a JSON array, which could potentially be hijacked and read by a malicious website. + +## Remediation + +JSON hijacking allows an attacker to send a GET request via a malicious web site or similar attack vector and utilize a user's stored credentials to retrieve sensitive or protected data to which that user has access. Since a JSON array on its own is valid JavaScript, a malicious GET request to a resource that returns only a JavaScript array can allow the attacker to use a malicious script to read the data in the array from the request. GET requests should never return a JSON array, even if the resource requires authentication to access. Consider using POST instead of a GET for this request or wrapping the array in a JSON object. + +## Links + +- [OWASP](https://owasp.org/Top10/A01_2021-Broken_Access_Control) +- [CWE](https://cwe.mitre.org/data/definitions/352.html) diff --git a/doc/user/application_security/api_security_testing/checks/json_injection_check.md b/doc/user/application_security/api_security_testing/checks/json_injection_check.md new file mode 100644 index 00000000000..264d77586a4 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/json_injection_check.md @@ -0,0 +1,20 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# JSON injection + +## Description + +Check for JSON serialization/injection vulnerabilities. + +## Remediation + +JSON injection is an attack technique used to manipulate or compromise the logic of a JSON application or service. The injection of unintended JSON content and/or structures into an JSON message can alter the intend logic of the application. Further, JSON injection can cause the insertion of malicious content into the resulting message/document. + +## Links + +- [OWASP](https://owasp.org/Top10/A03_2021-Injection) +- [CWE](CWE-929: OWASP Top Ten 2013 Category A1 - Injection) diff --git a/doc/user/application_security/api_security_testing/checks/open_redirect_check.md b/doc/user/application_security/api_security_testing/checks/open_redirect_check.md new file mode 100644 index 00000000000..d1670645cea --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/open_redirect_check.md @@ -0,0 +1,20 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Open redirect + +## Description + +Identify open redirects and determine if they can be abused by attackers. + +## Remediation + +Unvalidated redirects and forwards are possible when a web application accepts untrusted input that could cause the web application to redirect the request to a URL contained within untrusted input. By modifying untrusted URL input to a malicious site, an attacker may successfully launch a phishing scam and steal user credentials. Because the server name in the modified link is identical to the original site, phishing attempts may have a more trustworthy appearance. Unvalidated redirect and forward attacks can also be used to maliciously craft a URL that would pass the application’s access control check and then forward the attacker to privileged functions that they would normally not be able to access. + +## Links + +- [OWASP](https://owasp.org/Top10/A01_2021-Broken_Access_Control) +- [CWE](https://cwe.mitre.org/data/definitions/601.html) diff --git a/doc/user/application_security/api_security_testing/checks/os_command_injection_check.md b/doc/user/application_security/api_security_testing/checks/os_command_injection_check.md new file mode 100644 index 00000000000..ea399236a36 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/os_command_injection_check.md @@ -0,0 +1,32 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# OS command injection + +## Description + +Check for OS command injection vulnerabilities. An OS command injection attack consists of insertion or "injection" of an OS command via the input data from the client to the application. +A successful OS command injection exploit can run arbitrary commands. This allows an attacker the ability to read, write, and delete data. Depending on the user the commands run as, this can also include administrative functions. + +This check modifies parameters in the request (path, query string, headers, JSON, XML, etc.) to try and execute an OS command. Both standard injections and blind injections are performed. Blind injections cause delays in response when successful. + +## Remediation + +It is possible to execute arbitrary OS commands on the target application server. OS Command Injection is a critical vulnerability that can lead to a full system compromise. User input should never be used in constructing commands or command arguments to functions which execute OS commands. This includes filenames supplied by user uploads or downloads. + +Ensure your application does not: + +- Use user supplied information in the process name to execute. +- Use user supplied information in an OS command execution function which does +not escape shell meta-characters. +- Use user supplied information in arguments to OS commands. + +The application should have a hardcoded set of arguments that are to be passed to OS commands. If filenames are being passed to these functions, it is recommended that a hash of the filename be used instead, or some other unique identifier. It is strongly recommended that a native library that implements the same functionality be used instead of using OS system commands due to the risk of unknown attacks against third party commands. + +## Links + +- [OWASP](https://owasp.org/Top10/A03_2021-Injection) +- [CWE](https://cwe.mitre.org/data/definitions/78.html) diff --git a/doc/user/application_security/api_security_testing/checks/path_traversal_check.md b/doc/user/application_security/api_security_testing/checks/path_traversal_check.md new file mode 100644 index 00000000000..3fa5c16b184 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/path_traversal_check.md @@ -0,0 +1,30 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Path traversal + +## Description + +Many file operations are intended to take place within a restricted directory. By using special elements such as ".." and "/" separators, attackers can escape outside of the restricted location to access files or directories that are elsewhere on the system. One of the most common special elements is the "../" sequence, which in most modern operating systems is interpreted as the parent directory of the current location. This is referred to as relative path traversal. Path traversal also covers the use of absolute path-names such as "/usr/local/bin", which may also be useful in accessing unexpected files. This is referred to as absolute path traversal. + +In many programming languages, the injection of a null byte(the 0 or NUL) may allow an attacker to truncate a generated filename to widen the scope of attack. For example, the software may add ".txt" to any pathname, thus limiting the attacker to text files, but a null injection may effectively remove this restriction. + +This check modifies parameters in the request (path, query string, headers, JSON, XML, etc.) to try and access restricted files and files outside of the web-root. Logs and responses are then analyzed to try and detect if the file was successfully accessed. + +## Remediation + +The Path traversal attack technique allows an attacker access to files, directories, and commands that potentially reside outside the web document root directory. An attacker may manipulate a URL in such a way that the web site will execute or reveal the contents of arbitrary files anywhere on the web server. Any device that exposes an HTTP-based interface is potentially vulnerable to Path traversal. + +Most web sites restrict user access to a specific portion of the file-system, typically called the “web document root” or “CGI root” directory. These directories contain the files intended for user access and the executable necessary to drive web application functionality. To access files or execute commands anywhere on the file-system, Path traversal attacks will utilize the ability of special-characters sequences. + +The most basic Path traversal attack uses the “../” special-character sequence to alter the resource location requested in the URL. Although most popular web servers will prevent this technique from escaping the web document root, alternate encodings of the "../" sequence may help bypass the security filters. These method variations include valid and invalid Unicode-encoding ("..%u2216" or "..%c0%af") of the forward slash character, backslash characters ("..") on Windows-based servers, URL encoded characters "%2e%2e%2f"), and double URL encoding ("..%255c") of the backslash character. + +Even if the web server properly restricts Path traversal attempts in the URL path, a web application itself may still be vulnerable due to improper handling of user-supplied input. This is a common problem of web applications that use template mechanisms or load static text from files. In variations of the attack, the original URL parameter value is substituted with the file name of one of the web application's dynamic scripts. Consequently, the results can reveal source code because the file is interpreted as text instead of an executable script. These techniques often employ additional special characters such as the dot (".") to reveal the listing of the current working directory, or “%00” NULL characters in order to bypass rudimentary file extension checks. + +## Links + +- [OWASP](https://owasp.org/Top10/A01_2021-Broken_Access_Control) +- [CWE](https://cwe.mitre.org/data/definitions/22.html) diff --git a/doc/user/application_security/api_security_testing/checks/sensitive_file_disclosure_check.md b/doc/user/application_security/api_security_testing/checks/sensitive_file_disclosure_check.md new file mode 100644 index 00000000000..5c2f9ca0723 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/sensitive_file_disclosure_check.md @@ -0,0 +1,22 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Sensitive file disclosure + +## Description + +Check for sensitive file disclosure. This check looks for files that may contain sensitive information. Examples include .htaccess, .htpasswd, .bash_history, etc. + +## Remediation + +Information leakage is an application weakness where an application reveals sensitive data, such as technical details of the web application, environment, or user-specific data. Sensitive data may be used by an attacker to exploit the target web application, its hosting network, or its users. Therefore, leakage of sensitive data should be limited or prevented whenever possible. Information Leakage, in its most common form,is the result of one or more of the following conditions: A failure to scrub out HTML/Script comments containing sensitive information, improper application or server configurations, or differences in page responses for valid versus invalid data. + +In the case of this failure, one or more files and/or folders are accessable that should not be. This can include files common in home folders like such as command histories or files that contain secrets such as passwords. + +## Links + +- [OWASP](https://owasp.org/Top10/A01_2021-Broken_Access_Control) +- [CWE](https://cwe.mitre.org/data/definitions/200.html) diff --git a/doc/user/application_security/api_security_testing/checks/sensitive_information_disclosure_check.md b/doc/user/application_security/api_security_testing/checks/sensitive_information_disclosure_check.md new file mode 100644 index 00000000000..7ab04cda52c --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/sensitive_information_disclosure_check.md @@ -0,0 +1,22 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Sensitive information disclosure + +## Description + +Sensitive information disclosure check. This includes credit card numbers, health records, personal information, etc. + +## Remediation + +Sensitive information leakage is an application weakness where an application reveals sensitive, user-specific data. Sensitive data may be used by an attacker to exploit its users. Therefore, leakage of sensitive data should be limited or prevented whenever possible. Information Leakage, in its most common form, is the result of differences in page responses for valid versus invalid data. + +Pages that provide different responses based on the validity of the data can also lead to Information Leakage; specifically when data deemed confidential is being revealed as a result of the web application's design. Examples of sensitive data includes (but is not limited to): account numbers, user identifiers (Drivers license number, Passport number, Social Security Numbers, etc.) and user-specific information (passwords, sessions, addresses). Information Leakage in this context deals with exposure of key user data deemed confidential, or secret, that should not be exposed in plain view, even to the user. Credit card numbers and other heavily regulated information are prime examples of user data that needs to be further protected from exposure or leakage even with proper encryption and access controls already in place. + +## Links + +- [OWASP](https://owasp.org/Top10/A01_2021-Broken_Access_Control) +- [CWE](https://cwe.mitre.org/data/definitions/200.html) diff --git a/doc/user/application_security/api_security_testing/checks/session_cookie_check.md b/doc/user/application_security/api_security_testing/checks/session_cookie_check.md new file mode 100644 index 00000000000..0f4def9f74d --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/session_cookie_check.md @@ -0,0 +1,27 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Session cookie + +## Description + +Verify session cookie has correct flags and expiration. + +## Remediation + +Since HTTP is a stateless protocol, web sites commonly use cookies to store session IDs that uniquely identify a user from request to request. Consequently, each session ID's confidentiality must be maintained in order to prevent multiple users from accessing the same account. A stolen session ID can be used to view another user's account or perform a fraudulent +transaction. + +- One part of securing session ID's is to property mark them to expire and also require the correct set of flags to ensure they are not transmitted in the clear or accessible from scripting. +- HttpOnly is an additional flag included in a Set-Cookie HTTP response header. Using the HttpOnly flag when generating a cookie helps mitigate the risk of client side script accessing the protected cookie (if the browser supports it). If the HttpOnly flag (optional) is included in the HTTP response header, the cookie cannot be accessed through client side script (again if the browser supports this flag). As a result, even if a cross-site scripting (XSS) flaw exists, and a user accidentally accesses a link that exploits this flaw, the browser will not reveal the cookie to a third party. +- The Secure attribute for sensitive cookies in HTTPS sessions is not set, which could cause the user agent to send those cookies in plaintext over an HTTP session. +- A session related cookie was identified being used on an insecure transport protocol. Insecure transport protocols are those that do not make use of SSL/TLS to secure the connection. Examples of such protocols are 'http'. +- Insufficient Session Expiration occurs when a Web application permits an attacker to reuse old session credentials or session IDs for authorization. Insufficient Session Expiration increases a website's exposure to attacks that steal or reuse user's session identifiers. Since HTTP is a stateless protocol, websites commonly use cookies to store session IDs that uniquely identify a user from request to request. Consequently, each session ID's confidentiality must be maintained in order to prevent multiple users from accessing the same account. A stolen session ID can be used to view another user's account or perform a fraudulent transaction. One part of securing session ID's is to property mark them to expire and also require the correct set of flags to ensure they are not transmitted in the clear or accessible from scripting. + +## Links + +- [OWASP](https://owasp.org/Top10/A07_2021-Identification_and_Authentication_Failures) +- [CWE](https://cwe.mitre.org/data/definitions/930.html) diff --git a/doc/user/application_security/api_security_testing/checks/shellshock_check.md b/doc/user/application_security/api_security_testing/checks/shellshock_check.md new file mode 100644 index 00000000000..314554005f4 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/shellshock_check.md @@ -0,0 +1,20 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Shellshock + +## Description + +Check for Shellshock vulnerabilities. + +## Remediation + +Shellshock vulnerability takes advantage of a bug in BASH, in which, BASH incorrectly executes trailing commands when it imports a function definition stored into an environment variable. Any environment which allows defining BASH environmental variables could be vulnerable to this bug, as for example a Apache Web Server using mod_cgi and mod_cgid modules. A known-good request was modified to include malicious content. The malicious content includes an Shell shock attack in which the server-side application returns a specific text (evidence) in the response headers. + +## Links + +- [OWASP](https://owasp.org/Top10/A03_2021-Injection) +- [CWE](https://cwe.mitre.org/data/definitions/78.html) diff --git a/doc/user/application_security/api_security_testing/checks/sql_injection_check.md b/doc/user/application_security/api_security_testing/checks/sql_injection_check.md new file mode 100644 index 00000000000..c4b7671bdf9 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/sql_injection_check.md @@ -0,0 +1,24 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# SQL injection + +## Description + +Check for SQL and NoSQL injection vulnerabilities. A SQL injection attack consists of insertion or "injection" of a SQL query via the input data from the client to the application. A successful SQL injection exploit can read sensitive data from the database, modify database data (Insert/Update/Delete), execute administration operations on the database (such as shutdown the DBMS), recover the content of a given file present on the DBMS file system and in some cases issue commands to the operating system. SQL injection attacks are a type of injection attack, in which SQL commands are injected into data-plane input in order to effect the execution of predefined SQL commands. This check modifies parameters in the request (path, query string, headers, JSON, XML, etc.) to try and create a syntax error in the SQL or NoSQL query. Logs and responses are then analyzed to try and detect if an error occured. If an error is detected there is a high likelihood that a vulnerability exists. + +## Remediation + +The software constructs all or part of an SQL command using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the intended SQL command when it is sent to a downstream component. + +Without sufficient removal or quoting of SQL syntax in user-controllable inputs, the generated SQL query can cause those inputs to be interpreted as SQL instead of ordinary user data. This can be used to alter query logic to bypass security checks, or to insert additional statements that modify the back-end database, possibly including execution of system commands. + +SQL injection has become a common issue with database-driven websites. The flaw is easily detected, and easily exploited, and as such, any site or software package with even a minimal user base is likely to be subject to an attempted attack of this kind. This flaw depends on the fact that SQL makes no real distinction between the control and data planes. + +## Links + +- [OWASP](https://owasp.org/Top10/A03_2021-Injection) +- [CWE](https://cwe.mitre.org/data/definitions/930.html) diff --git a/doc/user/application_security/api_security_testing/checks/tls_server_configuration_check.md b/doc/user/application_security/api_security_testing/checks/tls_server_configuration_check.md new file mode 100644 index 00000000000..b7183ba177d --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/tls_server_configuration_check.md @@ -0,0 +1,29 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# TLS server configuration + +## Description + +Check for various TLS Server configuration issues. Checks TLS versions, hmacs, ciphers and compression algs supported by server. + +## Remediation + +Insufficient transport layer protection allows communication to be exposed to untrusted third-parties, providing an attack vector to compromise a web application and/or steal sensitive information. Websites typically use Secure Sockets Layer/Transport Layer Security (SSL/TLS) to provide encryption at the transport layer. However, unless the website is configured to use SSL/TLS and configured to use SSL/TLS properly, the website may be vulnerable to traffic interception and modification. + +SSL/TLS as a protocol have gone through several revisions over the years. Each new version adds features and fixes weaknesses in the protocol. Over time some versions of the protocol are broken so badly as to become vulnerabilities if supported. It's recommended to support only the most recent TLS versions such as TLS 1.3 (2018), and TLS 1.2 (2008). + +Compression has been linked to side-channel attacks on TLS connections. Disabling compression can prevent these attacks. One attack in particular, CRIME ("Compression Ratio Info-leak Made Easy") can be prevented. CRIME is an attack that targets clients, but if the server does not support compression the attack is mitigated. + +Historically, high grade cryptography was restricted from export to outside the United States. Because of this, websites were configured to support weak cryptographic options for those clients that were restricted to only using weak ciphers. Weak ciphers are vulnerable to attack because of the relative ease of breaking them; less than two weeks on a typical home computer +and a few seconds using dedicated hardware. + +Today, all modern browsers and websites use much stronger encryption, but some websites are still configured to support outdated weak ciphers. Because of this, an attacker may be able to force the client to downgrade to a weaker cipher when connecting to the website, allowing the attacker to break the weak encryption. For this reason, the server should be configured to only accept strong ciphers and not provide service to any client that requests using a weaker cipher. In addition, some websites are misconfigured to choose a weaker cipher even when the client will support a much stronger one. OWASP offers a guide to testing for SSL/TLS issues, including weak cipher support and misconfiguration, and there are other resources and tools as well. + +## Links + +- [OWASP](https://owasp.org/Top10/A02_2021-Cryptographic_Failures) +- [CWE](https://cwe.mitre.org/data/definitions/934.html) diff --git a/doc/user/application_security/api_security_testing/checks/xml_external_entity_check.md b/doc/user/application_security/api_security_testing/checks/xml_external_entity_check.md new file mode 100644 index 00000000000..e86076a9316 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/xml_external_entity_check.md @@ -0,0 +1,20 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# XML external entity + +## Description + +Check for XML DTD processing vulnerabilities. + +## Remediation + +XML external entity Attack is a type of attack against an application that parses XML input. This attack occurs when XML input containing a reference to an external entity is processed by a weakly configured XML parser. This attack may lead to the disclosure of confidential data, denial of service, server side request forgery, port scanning from the perspective of the machine where the parser is located, and other system impacts. + +## Links + +- [OWASP](https://owasp.org/Top10/A03_2021-Injection) +- [CWE](https://cwe.mitre.org/data/definitions/611.html) diff --git a/doc/user/application_security/api_security_testing/checks/xml_injection_check.md b/doc/user/application_security/api_security_testing/checks/xml_injection_check.md new file mode 100644 index 00000000000..d1bb4c6a8c9 --- /dev/null +++ b/doc/user/application_security/api_security_testing/checks/xml_injection_check.md @@ -0,0 +1,20 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# XML Injection Check + +## Description + +Check for XML serialization/injection vulnerabilities. + +## Remediation + +XML Injection is an attack technique used to manipulate or compromise the logic of an XML application or service. The injection of unintended XML content and/or structures into an XML message can alter the intend logic of the application. Further, XML injection can cause the insertion of malicious content into the resulting message/document. + +## Links + +- [OWASP](https://owasp.org/Top10/A03_2021-Injection) +- [CWE](https://cwe.mitre.org/data/definitions/91.html) diff --git a/doc/user/application_security/api_security_testing/configuration/customizing_analyzer_settings.md b/doc/user/application_security/api_security_testing/configuration/customizing_analyzer_settings.md new file mode 100644 index 00000000000..e63a02ac605 --- /dev/null +++ b/doc/user/application_security/api_security_testing/configuration/customizing_analyzer_settings.md @@ -0,0 +1,1014 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Customizing analyzer settings + +## Authentication + +Authentication is handled by providing the authentication token as a header or cookie. You can +provide a script that performs an authentication flow or calculates the token. + +### HTTP Basic Authentication + +[HTTP basic authentication](https://en.wikipedia.org/wiki/Basic_access_authentication) +is an authentication method built into the HTTP protocol and used in conjunction with +[transport layer security (TLS)](https://en.wikipedia.org/wiki/Transport_Layer_Security). + +We recommended that you [create a CI/CD variable](../../../../ci/variables/index.md#for-a-project) +for the password (for example, `TEST_API_PASSWORD`), and set it to be masked. You can create CI/CD +variables from the GitLab project's page at **Settings > CI/CD**, in the **Variables** section. +Because of the [limitations on masked variables](../../../../ci/variables/index.md#mask-a-cicd-variable), +you should Base64-encode the password before adding it as a variable. + +Finally, add two CI/CD variables to your `.gitlab-ci.yml` file: + +- `DAST_API_HTTP_USERNAME`: The username for authentication. +- `DAST_API_HTTP_PASSWORD_BASE64`: The Base64-encoded password for authentication. + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_HAR: test-api-recording.har + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_HTTP_USERNAME: testuser + DAST_API_HTTP_PASSWORD_BASE64: $TEST_API_PASSWORD +``` + +#### Raw password + +If you do not want to Base64-encode the password (or if you are using GitLab 15.3 or earlier) you can provide the raw password `DAST_API_HTTP_PASSWORD`, instead of using `DAST_API_HTTP_PASSWORD_BASE64`. + +### Bearer tokens + +Bearer tokens are used by several different authentication mechanisms, including OAuth2 and JSON Web +Tokens (JWT). Bearer tokens are transmitted using the `Authorization` HTTP header. To use Bearer +tokens with API security testing, you need one of the following: + +- A token that doesn't expire. +- A way to generate a token that lasts the length of testing. +- A Python script that API security testing can call to generate the token. + +#### Token doesn't expire + +If the Bearer token doesn't expire, use the `DAST_API_OVERRIDES_ENV` variable to provide it. This +variable's content is a JSON snippet that provides headers and cookies to add to outgoing HTTP requests for API security testing. + +Follow these steps to provide the Bearer token with `DAST_API_OVERRIDES_ENV`: + +1. [Create a CI/CD variable](../../../../ci/variables/index.md#for-a-project), + for example `TEST_API_BEARERAUTH`, with the value + `{"headers":{"Authorization":"Bearer dXNlcm5hbWU6cGFzc3dvcmQ="}}` (substitute your token). You + can create CI/CD variables from the GitLab projects page at **Settings > CI/CD**, in the + **Variables** section. + Due to the format of `TEST_API_BEARERAUTH` it's not possible to mask the variable. + To mask the token's value, you can create a second variable with the token values, and define + `TEST_API_BEARERAUTH` with the value `{"headers":{"Authorization":"Bearer $MASKED_VARIABLE"}}`. + +1. In your `.gitlab-ci.yml` file, set `DAST_API_OVERRIDES_ENV` to the variable you just created: + + ```yaml + stages: + - dast + + include: + - template: DAST-API.gitlab-ci.yml + + variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OVERRIDES_ENV: $TEST_API_BEARERAUTH + ``` + +1. To validate that authentication is working, run API security testing and review the job logs + and the test API's application logs. + +#### Token generated at test runtime + +If the Bearer token must be generated and doesn't expire during testing, you can provide API security testing with a file that has the token. A prior stage and job, or part of the API security testing job, can +generate this file. + +API security testing expects to receive a JSON file with the following structure: + +```json +{ + "headers" : { + "Authorization" : "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" + } +} +``` + +This file can be generated by a prior stage and provided to API security testing through the +`DAST_API_OVERRIDES_FILE` CI/CD variable. + +Set `DAST_API_OVERRIDES_FILE` in your `.gitlab-ci.yml` file: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OVERRIDES_FILE: dast-api-overrides.json +``` + +To validate that authentication is working, run API security testing and review the job logs and +the test API's application logs. + +#### Token has short expiration + +If the Bearer token must be generated and expires prior to the scan's completion, you can provide a +program or script for the API security testing scanner to execute on a provided interval. The provided script runs in +an Alpine Linux container that has Python 3 and Bash installed. If the Python script requires +additional packages, it must detect this and install the packages at runtime. + +The script must create a JSON file containing the Bearer token in a specific format: + +```json +{ + "headers" : { + "Authorization" : "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" + } +} +``` + +You must provide three CI/CD variables, each set for correct operation: + +- `DAST_API_OVERRIDES_FILE`: JSON file the provided command generates. +- `DAST_API_OVERRIDES_CMD`: Command that generates the JSON file. +- `DAST_API_OVERRIDES_INTERVAL`: Interval (in seconds) to run command. + +For example: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OVERRIDES_FILE: dast-api-overrides.json + DAST_API_OVERRIDES_CMD: renew_token.py + DAST_API_OVERRIDES_INTERVAL: 300 +``` + +To validate that authentication is working, run API security testing and review the job logs and the test API's application logs. See the [overrides section](#overrides) for more information about override commands. + +## Overrides + +API security testing provides a method to add or override specific items in your request, for example: + +- Headers +- Cookies +- Query string +- Form data +- JSON nodes +- XML nodes + +You can use this to inject semantic version headers, authentication, and so on. The +[authentication section](#authentication) includes examples of using overrides for that purpose. + +Overrides use a JSON document, where each type of override is represented by a JSON object: + +```json +{ + "headers": { + "header1": "value", + "header2": "value" + }, + "cookies": { + "cookie1": "value", + "cookie2": "value" + }, + "query": { + "query-string1": "value", + "query-string2": "value" + }, + "body-form": { + "form-param1": "value", + "form-param2": "value" + }, + "body-json": { + "json-path1": "value", + "json-path2": "value" + }, + "body-xml" : { + "xpath1": "value", + "xpath2": "value" + } +} +``` + +Example of setting a single header: + +```json +{ + "headers": { + "Authorization": "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" + } +} +``` + +Example of setting both a header and cookie: + +```json +{ + "headers": { + "Authorization": "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" + }, + "cookies": { + "flags": "677" + } +} +``` + +Example usage for setting a `body-form` override: + +```json +{ + "body-form": { + "username": "john.doe" + } +} +``` + +The override engine uses `body-form` when the request body has only form-data content. + +Example usage for setting a `body-json` override: + +```json +{ + "body-json": { + "$.credentials.access-token": "iddqd!42.$" + } +} +``` + +Each JSON property name in the object `body-json` is set to a [JSON Path](https://goessner.net/articles/JsonPath/) +expression. The JSON Path expression `$.credentials.access-token` identifies the node to be +overridden with the value `iddqd!42.$`. The override engine uses `body-json` when the request body +has only [JSON](https://www.json.org/json-en.html) content. + +For example, if the body is set to the following JSON: + +```json +{ + "credentials" : { + "username" :"john.doe", + "access-token" : "non-valid-password" + } +} +``` + +It is changed to: + +```json +{ + "credentials" : { + "username" :"john.doe", + "access-token" : "iddqd!42.$" + } +} +``` + +Here's an example for setting a `body-xml` override. The first entry overrides an XML attribute and +the second entry overrides an XML element: + +```json +{ + "body-xml" : { + "/credentials/@isEnabled": "true", + "/credentials/access-token/text()" : "iddqd!42.$" + } +} +``` + +Each JSON property name in the object `body-xml` is set to an +[XPath v2](https://www.w3.org/TR/xpath20/) +expression. The XPath expression `/credentials/@isEnabled` identifies the attribute node to override +with the value `true`. The XPath expression `/credentials/access-token/text()` identifies the +element node to override with the value `iddqd!42.$`. The override engine uses `body-xml` when the +request body has only [XML](https://www.w3.org/XML/) +content. + +For example, if the body is set to the following XML: + +```xml + + john.doe + non-valid-password + +``` + +It is changed to: + +```xml + + john.doe + iddqd!42.$ + +``` + +You can provide this JSON document as a file or environment variable. You may also provide a command +to generate the JSON document. The command can run at intervals to support values that expire. + +### Using a file + +To provide the overrides JSON as a file, the `DAST_API_OVERRIDES_FILE` CI/CD variable is set. The path is relative to the job current working directory. + +Here's an example `.gitlab-ci.yml`: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OVERRIDES_FILE: dast-api-overrides.json +``` + +### Using a CI/CD variable + +To provide the overrides JSON as a CI/CD variable, use the `DAST_API_OVERRIDES_ENV` variable. +This allows you to place the JSON as variables that can be masked and protected. + +In this example `.gitlab-ci.yml`, the `DAST_API_OVERRIDES_ENV` variable is set directly to the JSON: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OVERRIDES_ENV: '{"headers":{"X-API-Version":"2"}}' +``` + +In this example `.gitlab-ci.yml`, the `SECRET_OVERRIDES` variable provides the JSON. This is a +[group or instance level CI/CD variable defined in the UI](../../../../ci/variables/index.md#define-a-cicd-variable-in-the-ui): + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OVERRIDES_ENV: $SECRET_OVERRIDES +``` + +### Using a command + +If the value must be generated or regenerated on expiration, you can provide a program or script for +the API security testing scanner to execute on a specified interval. The provided command runs in an Alpine Linux +container that has Python 3 and Bash installed. + +You have to set the environment variable `DAST_API_OVERRIDES_CMD` to the program or script you would like +to execute. The provided command creates the overrides JSON file as defined previously. + +You might want to install other scripting runtimes like NodeJS or Ruby, or maybe you need to install a dependency for +your overrides command. In this case, we recommend setting the `DAST_API_PRE_SCRIPT` to the file path of a script which +provides those prerequisites. The script provided by `DAST_API_PRE_SCRIPT` is executed once, before the analyzer starts. + +See the [Alpine Linux package management](https://wiki.alpinelinux.org/wiki/Alpine_Linux_package_management) +page for information about installing Alpine Linux packages. + +You must provide three CI/CD variables, each set for correct operation: + +- `DAST_API_OVERRIDES_FILE`: File generated by the provided command. +- `DAST_API_OVERRIDES_CMD`: Overrides command in charge of generating the overrides JSON file periodically. +- `DAST_API_OVERRIDES_INTERVAL`: Interval in seconds to run command. + +Optionally: + +- `DAST_API_PRE_SCRIPT`: Script to install runtimes or dependencies before the scan starts. + +WARNING: +To execute scripts in Alpine Linux you must first use the command [`chmod`](https://www.gnu.org/software/coreutils/manual/html_node/chmod-invocation.html) to set the [execution permission](https://www.gnu.org/software/coreutils/manual/html_node/Setting-Permissions.html). For example, to set the execution permission of `script.py` for everyone, use the command: `chmod a+x script.py`. If needed, you can version your `script.py` with the execution permission already set. + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OVERRIDES_FILE: dast-api-overrides.json + DAST_API_OVERRIDES_CMD: renew_token.py + DAST_API_OVERRIDES_INTERVAL: 300 +``` + +### Debugging overrides + +By default the output of the overrides command is hidden. If the overrides command returns a non zero exit code, the command is displayed as part of your job output. Optionally, you can set the variable `DAST_API_OVERRIDES_CMD_VERBOSE` to any value to display overrides command output as it is generated. This is useful when testing your overrides script, but should be disabled afterwards as it slows down testing. + +It is also possible to write messages from your script to a log file that is collected when the job completes or fails. The log file must be created in a specific location and following a naming convention. + +Adding some basic logging to your overrides script is useful in case the script fails unexpectedly during standard running of the job. The log file is automatically included as an artifact of the job, allowing you to download it after the job has finished. + +Following our example, we provided `renew_token.py` in the environment variable `DAST_API_OVERRIDES_CMD`. Notice two things in the script: + +- Log file is saved in the location indicated by the environmental variable `CI_PROJECT_DIR`. +- Log filename should match `gl-*.log`. + +```python +#!/usr/bin/env python + +# Example of an overrides command + +# Override commands can update the overrides json file +# with new values to be used. This is a great way to +# update an authentication token that will expire +# during testing. + +import logging +import json +import os +import requests +import backoff + +# [1] Store log file in directory indicated by env var CI_PROJECT_DIR +working_directory = os.environ.get( 'CI_PROJECT_DIR') +overrides_file_name = os.environ.get('DAST_API_OVERRIDES_FILE', 'dast-api-overrides.json') +overrides_file_path = os.path.join(working_directory, overrides_file_name) + +# [2] File name should match the pattern: gl-*.log +log_file_path = os.path.join(working_directory, 'gl-user-overrides.log') + +# Set up logger +logging.basicConfig(filename=log_file_path, level=logging.DEBUG) + +# Use `backoff` decorator to retry in case of transient errors. +@backoff.on_exception(backoff.expo, + (requests.exceptions.Timeout, + requests.exceptions.ConnectionError), + max_time=30) +def get_auth_response(): + authorization_url = 'https://authorization.service/api/get_api_token' + return requests.get( + f'{authorization_url}', + auth=(os.environ.get('AUTH_USER'), os.environ.get('AUTH_PWD')) + ) + +# In our example, access token is retrieved from a given endpoint +try: + + # Performs a http request, response sample: + # { "Token" : "abcdefghijklmn" } + response = get_auth_response() + + # Check that the request is successful. may raise `requests.exceptions.HTTPError` + response.raise_for_status() + + # Gets JSON data + response_body = response.json() + +# If needed specific exceptions can be caught +# requests.ConnectionError : A network connection error problem occurred +# requests.HTTPError : HTTP request returned an unsuccessful status code. [Response.raise_for_status()] +# requests.ConnectTimeout : The request timed out while trying to connect to the remote server +# requests.ReadTimeout : The server did not send any data in the allotted amount of time. +# requests.TooManyRedirects : The request exceeds the configured number of maximum redirections +# requests.exceptions.RequestException : All exceptions that related to Requests +except json.JSONDecodeError as json_decode_error: + # logs errors related decoding JSON response + logging.error(f'Error, failed while decoding JSON response. Error message: {json_decode_error}') + raise +except requests.exceptions.RequestException as requests_error: + # logs exceptions related to `Requests` + logging.error(f'Error, failed while performing HTTP request. Error message: {requests_error}') + raise +except Exception as e: + # logs any other error + logging.error(f'Error, unknown error while retrieving access token. Error message: {e}') + raise + +# computes object that holds overrides file content. +# It uses data fetched from request +overrides_data = { + "headers": { + "Authorization": f"Token {response_body['Token']}" + } +} + +# log entry informing about the file override computation +logging.info("Creating overrides file: %s" % overrides_file_path) + +# attempts to overwrite the file +try: + if os.path.exists(overrides_file_path): + os.unlink(overrides_file_path) + + # overwrites the file with our updated dictionary + with open(overrides_file_path, "wb+") as fd: + fd.write(json.dumps(overrides_data).encode('utf-8')) +except Exception as e: + # logs any other error + logging.error(f'Error, unknown error when overwriting file {overrides_file_path}. Error message: {e}') + raise + +# logs informing override has finished successfully +logging.info("Override file has been updated") + +# end +``` + +In the overrides command example, the Python script depends on the `backoff` library. To make sure the library is installed before executing the Python script, the `DAST_API_PRE_SCRIPT` is set to a script that installs the dependencies of your overrides command. +As for example, the following script `user-pre-scan-set-up.sh` + +```shell +#!/bin/bash + +# user-pre-scan-set-up.sh +# Ensures python dependencies are installed + +echo "**** install python dependencies ****" + +python3 -m ensurepip +pip3 install --no-cache --upgrade \ + pip \ + backoff + +echo "**** python dependencies installed ****" + +# end +``` + +You have to update your configuration to set the `DAST_API_PRE_SCRIPT` to our new `user-pre-scan-set-up.sh` script. For example: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_PRE_SCRIPT: user-pre-scan-set-up.sh + DAST_API_OVERRIDES_FILE: dast-api-overrides.json + DAST_API_OVERRIDES_CMD: renew_token.py + DAST_API_OVERRIDES_INTERVAL: 300 +``` + +In the previous sample, you could use the script `user-pre-scan-set-up.sh` to also install new runtimes or applications that later on you could use in our overrides command. + +## Request Headers + +The request headers feature lets you specify fixed values for the headers during the scan session. For example, you can use the configuration variable `DAST_API_REQUEST_HEADERS` to set a fixed value in the `Cache-Control` header. If the headers you need to set include sensitive values like the `Authorization` header, use the [masked variable](../../../../ci/variables/index.md#mask-a-cicd-variable) feature along with the [variable `DAST_API_REQUEST_HEADERS_BASE64`](#base64). + +If the `Authorization` header or any other header needs to get updated while the scan is in progress, consider using the [overrides](#overrides) feature. + +The variable `DAST_API_REQUEST_HEADERS` lets you specify a comma-separated (`,`) list of headers. These headers are included on each request that the scanner performs. Each header entry in the list consists of a name followed by a colon (`:`) and then by its value. Whitespace before the key or value is ignored. For example, to declare a header name `Cache-Control` with the value `max-age=604800`, the header entry is `Cache-Control: max-age=604800`. To use two headers, `Cache-Control: max-age=604800` and `Age: 100`, set `DAST_API_REQUEST_HEADERS` variable to `Cache-Control: max-age=604800, Age: 100`. + +The order in which the different headers are provided into the variable `DAST_API_REQUEST_HEADERS` does not affect the result. Setting `DAST_API_REQUEST_HEADERS` to `Cache-Control: max-age=604800, Age: 100` produces the same result as setting it to `Age: 100, Cache-Control: max-age=604800`. + +### Base64 + +The `DAST_API_REQUEST_HEADERS_BASE64` variable accepts the same list of headers as `DAST_API_REQUEST_HEADERS`, with the only difference that the entire value of the variable must be Base64-encoded. For example, to set `DAST_API_REQUEST_HEADERS_BASE64` variable to `Authorization: QmVhcmVyIFRPS0VO, Cache-control: bm8tY2FjaGU=`, ensure you convert the list to its Base64 equivalent: `QXV0aG9yaXphdGlvbjogUW1WaGNtVnlJRlJQUzBWTywgQ2FjaGUtY29udHJvbDogYm04dFkyRmphR1U9`, and the Base64-encoded value must be used. This is useful when storing secret header values in a [masked variable](../../../../ci/variables/index.md#mask-a-cicd-variable), which has character set restrictions. + +WARNING: +Base64 is used to support the [masked variable](../../../../ci/variables/index.md#mask-a-cicd-variable) feature. Base64 encoding is not by itself a security measure, because sensitive values can be easily decoded. + +### Example: Adding a list of headers on each request using plain text + +In the following example of a `.gitlab-ci.yml`, `DAST_API_REQUEST_HEADERS` configuration variable is set to provide two header values as explained in [request headers](#request-headers). + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_REQUEST_HEADERS: 'Cache-control: no-cache, Save-Data: on' +``` + +### Example: Using a masked CI/CD variable + +The following `.gitlab-ci.yml` sample assumes the [masked variable](../../../../ci/variables/index.md#mask-a-cicd-variable) `SECRET_REQUEST_HEADERS_BASE64` is defined as a [group or instance level CI/CD variable defined in the UI](../../../../ci/variables/index.md#define-a-cicd-variable-in-the-ui). The value of `SECRET_REQUEST_HEADERS_BASE64` is set to `WC1BQ01FLVNlY3JldDogc31jcnt0ISwgWC1BQ01FLVRva2VuOiA3MDVkMTZmNWUzZmI=`, which is the Base64-encoded text version of `X-ACME-Secret: s3cr3t!, X-ACME-Token: 705d16f5e3fb`. Then, it can be used as follows: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_REQUEST_HEADERS_BASE64: $SECRET_REQUEST_HEADERS_BASE64 +``` + +Consider using `DAST_API_REQUEST_HEADERS_BASE64` when storing secret header values in a [masked variable](../../../../ci/variables/index.md#mask-a-cicd-variable), which has character set restrictions. + +## Exclude Paths + +When testing an API it can be useful to exclude certain paths. For example, you might exclude testing of an authentication service or an older version of the API. To exclude paths, use the `DAST_API_EXCLUDE_PATHS` CI/CD variable . This variable is specified in your `.gitlab-ci.yml` file. To exclude multiple paths, separate entries using the `;` character. In the provided paths you can use a single character wildcard `?` and `*` for a multiple character wildcard. + +To verify the paths are excluded, review the `Tested Operations` and `Excluded Operations` portion of the job output. You should not see any excluded paths listed under `Tested Operations`. + +```plaintext +2021-05-27 21:51:08 [INF] DAST API: --[ Tested Operations ]------------------------- +2021-05-27 21:51:08 [INF] DAST API: 201 POST http://target:7777/api/users CREATED +2021-05-27 21:51:08 [INF] DAST API: ------------------------------------------------ +2021-05-27 21:51:08 [INF] DAST API: --[ Excluded Operations ]----------------------- +2021-05-27 21:51:08 [INF] DAST API: GET http://target:7777/api/messages +2021-05-27 21:51:08 [INF] DAST API: POST http://target:7777/api/messages +2021-05-27 21:51:08 [INF] DAST API: ------------------------------------------------ +``` + +### Examples + +This example excludes the `/auth` resource. This does not exclude child resources (`/auth/child`). + +```yaml +variables: + DAST_API_EXCLUDE_PATHS: /auth +``` + +To exclude `/auth`, and child resources (`/auth/child`), we use a wildcard. + +```yaml +variables: + DAST_API_EXCLUDE_PATHS: /auth* +``` + +To exclude multiple paths we use the `;` character. In this example we exclude `/auth*` and `/v1/*`. + +```yaml +variables: + DAST_API_EXCLUDE_PATHS: /auth*;/v1/* +``` + +To exclude one or more nested levels within a path we use `**`. In this example we are testing API endpoints. We are testing `/api/v1/` and `/api/v2/` of a data query requesting `mass`, `brightness` and `coordinates` data for `planet`, `moon`, `star`, and `satellite` objects. Example paths that could be scanned include, but are not limited to: + +- `/api/v2/planet/coordinates` +- `/api/v1/star/mass` +- `/api/v2/satellite/brightness` + +In this example we test the `brightness` endpoint only: + +```yaml +variables: + DAST_API_EXCLUDE_PATHS: /api/**/mass;/api/**/coordinates +``` + +### Exclude parameters + +While testing an API you may might want to exclude a parameter (query string, header, or body element) from testing. This may be needed because a parameter always causes a failure, slows down testing, or for other reasons. To exclude parameters, you can set one of the following variables: `DAST_API_EXCLUDE_PARAMETER_ENV` or `DAST_API_EXCLUDE_PARAMETER_FILE`. + +The `DAST_API_EXCLUDE_PARAMETER_ENV` allows providing a JSON string containing excluded parameters. This is a good option if the JSON is short and does not change often. Another option is the variable `DAST_API_EXCLUDE_PARAMETER_FILE`. This variable is set to a file path that can be checked into the repository, created by another job as an artifact, or generated at runtime with a pre-script using `DAST_API_PRE_SCRIPT`. + +#### Exclude parameters using a JSON document + +The JSON document contains a JSON object, this object uses specific properties to identify which parameter should be excluded. +You can provide the following properties to exclude specific parameters during the scanning process: + +- `headers`: Use this property to exclude specific headers. The property's value is an array of header names to be excluded. Names are case-insensitive. +- `cookies`: Use this property's value to exclude specific cookies. The property's value is an array of cookie names to be excluded. Names are case-sensitive. +- `query`: Use this property to exclude specific fields from the query string. The property's value is an array of field names from the query string to be excluded. Names are case-sensitive. +- `body-form`: Use this property to exclude specific fields from a request that uses the media type `application/x-www-form-urlencoded`. The property's value is an array of the field names from the body to be excluded. Names are case-sensitive. +- `body-json`: Use this property to exclude specific JSON nodes from a request that uses the media type `application/json`. The property's value is an array, each entry of the array is a [JSON Path](https://goessner.net/articles/JsonPath/) expression. +- `body-xml`: Use this property to exclude specific XML nodes from a request that uses media type `application/xml`. The property's value is an array, each entry of the array is a [XPath v2](https://www.w3.org/TR/xpath20/) expression. + +Thus, the following JSON document is an example of the expected structure to exclude parameters. + +```json +{ + "headers": [ + "header1", + "header2" + ], + "cookies": [ + "cookie1", + "cookie2" + ], + "query": [ + "query-string1", + "query-string2" + ], + "body-form": [ + "form-param1", + "form-param2" + ], + "body-json": [ + "json-path-expression-1", + "json-path-expression-2" + ], + "body-xml" : [ + "xpath-expression-1", + "xpath-expression-2" + ] +} +``` + +#### Examples + +##### Excluding a single header + +To exclude the header `Upgrade-Insecure-Requests`, set the `header` property's value to an array with the header name: `[ "Upgrade-Insecure-Requests" ]`. For instance, the JSON document looks like this: + +```json +{ + "headers": [ "Upgrade-Insecure-Requests" ] +} +``` + +Header names are case-insensitive, so the header name `UPGRADE-INSECURE-REQUESTS` is equivalent to `Upgrade-Insecure-Requests`. + +##### Excluding both a header and two cookies + +To exclude the header `Authorization`, and the cookies `PHPSESSID` and `csrftoken`, set the `headers` property's value to an array with header name `[ "Authorization" ]` and the `cookies` property's value to an array with the cookies' names `[ "PHPSESSID", "csrftoken" ]`. For instance, the JSON document looks like this: + +```json +{ + "headers": [ "Authorization" ], + "cookies": [ "PHPSESSID", "csrftoken" ] +} +``` + +##### Excluding a `body-form` parameter + +To exclude the `password` field in a request that uses `application/x-www-form-urlencoded`, set the `body-form` property's value to an array with the field name `[ "password" ]`. For instance, the JSON document looks like this: + +```json +{ + "body-form": [ "password" ] +} +``` + +The exclude parameters uses `body-form` when the request uses a content type `application/x-www-form-urlencoded`. + +##### Excluding a specific JSON nodes using JSON Path + +To exclude the `schema` property in the root object, set the `body-json` property's value to an array with the JSON Path expression `[ "$.schema" ]`. + +The JSON Path expression uses special syntax to identify JSON nodes: `$` refers to the root of the JSON document, `.` refers to the current object (in our case the root object), and the text `schema` refers to a property name. Thus, the JSON path expression `$.schema` refers to a property `schema` in the root object. +For instance, the JSON document looks like this: + +```json +{ + "body-json": [ "$.schema" ] +} +``` + +The exclude parameters uses `body-json` when the request uses a content type `application/json`. Each entry in `body-json` is expected to be a [JSON Path expression](https://goessner.net/articles/JsonPath/). In JSON Path characters like `$`, `*`, `.` among others have special meaning. + +##### Excluding multiple JSON nodes using JSON Path + +To exclude the property `password` on each entry of an array of `users` at the root level, set the `body-json` property's value to an array with the JSON Path expression `[ "$.users[*].paswword" ]`. + +The JSON Path expression starts with `$` to refer to the root node and uses `.` to refer to the current node. Then, it uses `users` to refer to a property and the characters `[` and `]` to enclose the index in the array you want to use, instead of providing a number as an index you use `*` to specify any index. After the index reference, we find `.` which now refers to any given selected index in the array, preceded by a property name `password`. + +For instance, the JSON document looks like this: + +```json +{ + "body-json": [ "$.users[*].paswword" ] +} +``` + +The exclude parameters uses `body-json` when the request uses a content type `application/json`. Each entry in `body-json` is expected to be a [JSON Path expression](https://goessner.net/articles/JsonPath/). In JSON Path characters like `$`, `*`, `.` among others have special meaning. + +##### Excluding a XML attribute + +To exclude an attribute named `isEnabled` located in the root element `credentials`, set the `body-xml` property's value to an array with the XPath expression `[ "/credentials/@isEnabled" ]`. + +The XPath expression `/credentials/@isEnabled`, starts with `/` to indicate the root of the XML document, then it is followed by the word `credentials` which indicates the name of the element to match. It uses a `/` to refer to a node of the previous XML element, and the character `@` to indicate that the name `isEnable` is an attribute. + +For instance, the JSON document looks like this: + +```json +{ + "body-xml": [ + "/credentials/@isEnabled" + ] +} +``` + +The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. + +##### Excluding a XML text's element + +To exclude the text of the `username` element contained in root node `credentials`, set the `body-xml` property's value to an array with the XPath expression `[/credentials/username/text()" ]`. + +In the XPath expression `/credentials/username/text()`, the first character `/` refers to the root XML node, and then after it indicates an XML element's name `credentials`. Similarly, the character `/` refers to the current element, followed by a new XML element's name `username`. Last part has a `/` that refers to the current element, and uses a XPath function called `text()` which identifies the text of the current element. + +For instance, the JSON document looks like this: + +```json +{ + "body-xml": [ + "/credentials/username/text()" + ] +} +``` + +The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. + +##### Excluding an XML element + +To exclude the element `username` contained in root node `credentials`, set the `body-xml` property's value to an array with the XPath expression `[/credentials/username" ]`. + +In the XPath expression `/credentials/username`, the first character `/` refers to the root XML node, and then after it indicates an XML element's name `credentials`. Similarly, the character `/` refers to the current element, followed by a new XML element's name `username`. + +For instance, the JSON document looks like this: + +```json +{ + "body-xml": [ + "/credentials/username" + ] +} +``` + +The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. + +##### Excluding an XML node with namespaces + +To exclude anXML element `login` which is defined in namespace `s`, and contained in `credentials` root node, set the `body-xml` property's value to an array with the XPath expression `[ "/credentials/s:login" ]`. + +In the XPath expression `/credentials/s:login`, the first character `/` refers to the root XML node, and then after it indicates an XML element's name `credentials`. Similarly, the character `/` refers to the current element, followed by a new XML element's name `s:login`. Notice that name contains the character `:`, this character separates the namespace from the node name. + +The namespace name should have been defined in the XML document which is part of the body request. You may check the namespace in the specification document HAR, OpenAPI, or Postman Collection file. + +```json +{ + "body-xml": [ + "/credentials/s:login" + ] +} +``` + +The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be an [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath, expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. + +#### Using a JSON string + +To provide the exclusion JSON document set the variable `DAST_API_EXCLUDE_PARAMETER_ENV` with the JSON string. In the following example, the `.gitlab-ci.yml`, the `DAST_API_EXCLUDE_PARAMETER_ENV` variable is set to a JSON string: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_EXCLUDE_PARAMETER_ENV: '{ "headers": [ "Upgrade-Insecure-Requests" ] }' +``` + +#### Using a file + +To provide the exclusion JSON document set the variable `DAST_API_EXCLUDE_PARAMETER_FILE` with the JSON file path. The file path is relative to the job current working directory. In the following example `.gitlab-ci.yml` content, the `DAST_API_EXCLUDE_PARAMETER_FILE` variable is set to a JSON file path: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_EXCLUDE_PARAMETER_FILE: dast-api-exclude-parameters.json +``` + +The `dast-api-exclude-parameters.json` is a JSON document that follows the structure of [exclude parameters document](#exclude-parameters-using-a-json-document). + +### Exclude URLs + +As an alternative to excluding by paths, you can filter by any other component in the URL by using the `DAST_API_EXCLUDE_URLS` CI/CD variable. This variable can be set in your `.gitlab-ci.yml` file. The variable can store multiple values, separated by commas (`,`). Each value is a regular expression. Because each entry is a regular expression, an entry like `.*` excludes all URLs because it is a regular expression that matches everything. + +In your job output you can check if any URLs matched any provided regular expression from `DAST_API_EXCLUDE_URLS`. Matching operations are listed in the **Excluded Operations** section. Operations listed in the **Excluded Operations** should not be listed in the **Tested Operations** section. For example the following portion of a job output: + +```plaintext +2021-05-27 21:51:08 [INF] DAST API: --[ Tested Operations ]------------------------- +2021-05-27 21:51:08 [INF] DAST API: 201 POST http://target:7777/api/users CREATED +2021-05-27 21:51:08 [INF] DAST API: ------------------------------------------------ +2021-05-27 21:51:08 [INF] DAST API: --[ Excluded Operations ]----------------------- +2021-05-27 21:51:08 [INF] DAST API: GET http://target:7777/api/messages +2021-05-27 21:51:08 [INF] DAST API: POST http://target:7777/api/messages +2021-05-27 21:51:08 [INF] DAST API: ------------------------------------------------ +``` + +NOTE: +Each value in `DAST_API_EXCLUDE_URLS` is a regular expression. Characters such as `.` , `*` and `$` among many others have special meanings in [regular expressions](https://en.wikipedia.org/wiki/Regular_expression#Standards). + +#### Examples + +##### Excluding a URL and child resources + +The following example excludes the URL `http://target/api/auth` and its child resources. + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_TARGET_URL: http://target/ + DAST_API_OPENAPI: test-api-specification.json + DAST_API_EXCLUDE_URLS: http://target/api/auth +``` + +##### Excluding two URLs and allow their child resources + +To exclude the URLs `http://target/api/buy` and `http://target/api/sell` but allowing to scan their child resources, for instance: `http://target/api/buy/toy` or `http://target/api/sell/chair`. You could use the value `http://target/api/buy/$,http://target/api/sell/$`. This value is using two regular expressions, each of them separated by a `,` character. Hence, it contains `http://target/api/buy$` and `http://target/api/sell$`. In each regular expression, the trailing `$` character points out where the matching URL should end. + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_TARGET_URL: http://target/ + DAST_API_OPENAPI: test-api-specification.json + DAST_API_EXCLUDE_URLS: http://target/api/buy/$,http://target/api/sell/$ +``` + +##### Excluding two URLs and their child resources + +To exclude the URLs: `http://target/api/buy` and `http://target/api/sell`, and their child resources. To provide multiple URLs we use the `,` character as follows: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_TARGET_URL: http://target/ + DAST_API_OPENAPI: test-api-specification.json + DAST_API_EXCLUDE_URLS: http://target/api/buy,http://target/api/sell +``` + +##### Excluding URL using regular expressions + +To exclude exactly `https://target/api/v1/user/create` and `https://target/api/v2/user/create` or any other version (`v3`,`v4`, and more). We could use `https://target/api/v.*/user/create$`, in the previous regular expression `.` indicates any character and `*` indicates zero or more times, additionally `$` indicates that the URL should end there. + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_TARGET_URL: http://target/ + DAST_API_OPENAPI: test-api-specification.json + DAST_API_EXCLUDE_URLS: https://target/api/v.*/user/create$ +``` diff --git a/doc/user/application_security/api_security_testing/configuration/enabling_the_analyzer.md b/doc/user/application_security/api_security_testing/configuration/enabling_the_analyzer.md new file mode 100644 index 00000000000..2107d862bfd --- /dev/null +++ b/doc/user/application_security/api_security_testing/configuration/enabling_the_analyzer.md @@ -0,0 +1,937 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Enabling the analyzer + +You can specify the API you want to scan by using: + +- [OpenAPI v2 or v3 Specification](#openapi-specification) +- [GraphQL Schema](#graphql-schema) +- [HTTP Archive (HAR)](#http-archive-har) +- [Postman Collection v2.0 or v2.1](#postman-collection) + +## OpenAPI Specification + +The [OpenAPI Specification](https://www.openapis.org/) (formerly the Swagger Specification) is an API description format for REST APIs. +This section shows you how to configure API security testing scanning using an OpenAPI Specification to provide information about the target API to test. +OpenAPI Specifications are provided as a file system resource or URL. Both JSON and YAML OpenAPI formats are supported. + +API security testing uses an OpenAPI document to generate the request body. When a request body is required, +the body generation is limited to these body types: + +- `application/x-www-form-urlencoded` +- `multipart/form-data` +- `application/json` +- `application/xml` + +## OpenAPI and media types + +A media type (formerly known as MIME type) is an identifier for file formats and format contents transmitted. A OpenAPI document lets you specify that a given operation can accept different media types, hence a given request can send data using different file content. As for example, a `PUT /user` operation to update user data could accept data in either XML (media type `application/xml`) or JSON (media type `application/json`) format. +OpenAPI 2.x lets you specify the accepted media types globally or per operation, and OpenAPI 3.x lets you specify the accepted media types per operation. API security testing will check the listed media types, and try to produce sample data for each supported media type. + +- The default behavior is to select one of the supported media types to use. The first supported media type is chosen from the list. This behavior is configurable. + +Testing the same operation (for example, `POST /user`) using different media types (for example, `application/json` and `application/xml`) is not always desirable. +For example, if the target application executes the same code regardless of the request content type, it will take longer to finish the test session, and it may report duplicated vulnerabilities related to the request body depending on the target app. + +The environment variable `DAST_API_OPENAPI_ALL_MEDIA_TYPES` lets you specify whether or not to use all supported media types instead of one when generating requests for a given operation. When the environment variable `DAST_API_OPENAPI_ALL_MEDIA_TYPES` is set to any value, API security testing tries to generate requests for all supported media types instead of one in a given operation. This will cause testing to take longer as testing is repeated for each provided media type. + +Alternatively, the variable `DAST_API_OPENAPI_MEDIA_TYPES` is used to provide a list of media types that will each be tested. Providing more than one media type causes testing to take longer, as testing is performed for each media type selected. When the environment variable `DAST_API_OPENAPI_MEDIA_TYPES` is set to a list of media types, only the listed media types are included when creating requests. + +Multiple media types in `DAST_API_OPENAPI_MEDIA_TYPES` are separated by a colon (`:`). For example, to limit request generation to the media types `application/x-www-form-urlencoded` and `multipart/form-data`, set the environment variable `DAST_API_OPENAPI_MEDIA_TYPES` to `application/x-www-form-urlencoded:multipart/form-data`. Only supported media types in this list are included when creating requests, though non-supported media types are always skipped. A media type text may contain different sections. For example, `application/vnd.api+json; charset=UTF-8`, is a compound of `type "/" [tree "."] subtype ["+" suffix]* [";" parameter]`. Parameters are not taken into account when performing the filtering media types on request generation. + +The environment variables `DAST_API_OPENAPI_ALL_MEDIA_TYPES` and `DAST_API_OPENAPI_MEDIA_TYPES` allow you to decide how to handle media types. These settings are mutually exclusive. If both are enabled, API security testing reports an error. + +### Configure API security testing with an OpenAPI Specification + +To configure API security testing scanning with an OpenAPI Specification: + +1. [Include](../../../../ci/yaml/index.md#includetemplate) + the [`DAST-API.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml) in your `.gitlab-ci.yml` file. + +1. The [configuration file](variables.md#configuration-files) has several testing profiles defined with different checks enabled. We recommend that you start with the `Quick` profile. + Testing with this profile completes faster, allowing for easier configuration validation. + Provide the profile by adding the `DAST_API_PROFILE` CI/CD variable to your `.gitlab-ci.yml` file. + +1. Provide the location of the OpenAPI Specification as either a file or URL. + Specify the location by adding the `DAST_API_OPENAPI` variable. + +1. The target API instance's base URL is also required. Provide it by using the `DAST_API_TARGET_URL` + variable or an `environment_url.txt` file. + + Adding the URL in an `environment_url.txt` file at your project's root is great for testing in + dynamic environments. To run API security testing against an app dynamically created during a GitLab CI/CD + pipeline, have the app persist its URL in an `environment_url.txt` file. API security testing + automatically parses that file to find its scan target. You can see an + [example of this in our Auto DevOps CI YAML](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml). + +Complete example configuration of using an OpenAPI Specification: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_OPENAPI: test-api-specification.json + DAST_API_TARGET_URL: http://test-deployment/ +``` + +This is a minimal configuration for API security testing. From here you can: + +- [Run your first scan](#running-your-first-scan). +- [Add authentication](customizing_analyzer_settings.md#authentication). +- Learn how to [handle false positives](#handling-false-positives). + +## HTTP Archive (HAR) + +The [HTTP Archive format (HAR)](../../api_fuzzing/create_har_files.md) is an archive file format for +logging HTTP transactions. When used with the GitLab API security testing scanner, the HAR file must contain +records of calling the web API to test. The API security testing scanner extracts all of the requests and uses them +to perform testing. + +You can use various tools to generate HAR files: + +- [Insomnia Core](https://insomnia.rest/): API client +- [Chrome](https://www.google.com/chrome/): Browser +- [Firefox](https://www.mozilla.org/en-US/firefox/): Browser +- [Fiddler](https://www.telerik.com/fiddler): Web debugging proxy +- [GitLab HAR Recorder](https://gitlab.com/gitlab-org/security-products/har-recorder): Command line + +WARNING: +HAR files may contain sensitive information such as authentication tokens, API keys, and session +cookies. We recommend that you review the HAR file contents before adding them to a repository. + +### API security testing scanning with a HAR file + +To configure API security testing to use a HAR file that provides information about the target API to test: + +1. [Include](../../../../ci/yaml/index.md#includetemplate) + the [`DAST-API.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml) in your `.gitlab-ci.yml` file. + +1. The [configuration file](variables.md#configuration-files) has several testing profiles defined with different checks enabled. We recommend that you start with the `Quick` profile. + Testing with this profile completes faster, allowing for easier configuration validation. + + Provide the profile by adding the `DAST_API_PROFILE` CI/CD variable to your `.gitlab-ci.yml` file. + +1. Provide the location of the HAR file. You can provide the location as a file path + or URL. Specify the location by adding the `DAST_API_HAR` variable. + +1. The target API instance's base URL is also required. Provide it by using the `DAST_API_TARGET_URL` + variable or an `environment_url.txt` file. + + Adding the URL in an `environment_url.txt` file at your project's root is great for testing in + dynamic environments. To run API security testing against an app dynamically created during a GitLab CI/CD + pipeline, have the app persist its URL in an `environment_url.txt` file. API security testing + automatically parses that file to find its scan target. You can see an + [example of this in our Auto DevOps CI YAML](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml). + +Complete example configuration of using an HAR file: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_HAR: test-api-recording.har + DAST_API_TARGET_URL: http://test-deployment/ +``` + +This example is a minimal configuration for API security testing. From here you can: + +- [Run your first scan](#running-your-first-scan). +- [Add authentication](customizing_analyzer_settings.md#authentication). +- Learn how to [handle false positives](#handling-false-positives). + +## GraphQL Schema + +> - Support for GraphQL Schema was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352780) in GitLab 15.4. + +GraphQL is a query language for your API and an alternative to REST APIs. +API security testing supports testing GraphQL endpoints multiple ways: + +- Test using the GraphQL Schema. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352780) in GitLab 15.4. +- Test using a recording (HAR) of GraphQL queries. +- Test using a Postman Collection containing GraphQL queries. + +This section documents how to test using a GraphQL schema. The GraphQL schema support in +API security testing is able to query the schema from endpoints that support [introspection](https://graphql.org/learn/introspection/). +Introspection is enabled by default to allow tools like GraphiQL to work. +For details on how to enable introspection, see your GraphQL framework documentation. + +### API security testing scanning with a GraphQL endpoint URL + +The GraphQL support in API security testing is able to query a GraphQL endpoint for the schema. + +NOTE: +The GraphQL endpoint must support introspection queries for this method to work correctly. + +To configure API security testing to use a GraphQL endpoint URL that provides information about the target API to test: + +1. [Include](../../../../ci/yaml/index.md#includetemplate) + the [`DAST-API.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml) in your `.gitlab-ci.yml` file. + +1. Provide the path to the GraphQL endpoint, for example `/api/graphql`. Specify the location by adding the `DAST_API_GRAPHQL` variable. + +1. The target API instance's base URL is also required. Provide it by using the `DAST_API_TARGET_URL` + variable or an `environment_url.txt` file. + + Adding the URL in an `environment_url.txt` file at your project's root is great for testing in + dynamic environments. See the [dynamic environment solutions](../troubleshooting.md#dynamic-environment-solutions) section of our documentation for more information. + +Complete example configuration of using a GraphQL endpoint path: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +dast_api: + variables: + DAST_API_GRAPHQL: /api/graphql + DAST_API_TARGET_URL: http://test-deployment/ +``` + +This example is a minimal configuration for API security testing. From here you can: + +- [Run your first scan](#running-your-first-scan). +- [Add authentication](customizing_analyzer_settings.md#authentication). +- Learn how to [handle false positives](#handling-false-positives). + +### API security testing scanning with a GraphQL Schema file + +API security testing can use a GraphQL schema file to understand and test a GraphQL endpoint that has introspection disabled. To use a GraphQL schema file, it must be in the introspection JSON format. A GraphQL schema can be converted to a the introspection JSON format using an online 3rd party tool: [https://transform.tools/graphql-to-introspection-json](https://transform.tools/graphql-to-introspection-json). + +To configure API security testing to use a GraphQL schema file that provides information about the target API to test: + +1. [Include](../../../../ci/yaml/index.md#includetemplate) + the [`DAST-API.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml) in your `.gitlab-ci.yml` file. + +1. Provide the GraphQL endpoint path, for example `/api/graphql`. Specify the path by adding the `DAST_API_GRAPHQL` variable. + +1. Provide the location of the GraphQL schema file. You can provide the location as a file path + or URL. Specify the location by adding the `DAST_API_GRAPHQL_SCHEMA` variable. + +1. The target API instance's base URL is also required. Provide it by using the `DAST_API_TARGET_URL` + variable or an `environment_url.txt` file. + + Adding the URL in an `environment_url.txt` file at your project's root is great for testing in + dynamic environments. See the [dynamic environment solutions](../troubleshooting.md#dynamic-environment-solutions) section of our documentation for more information. + +Complete example configuration of using an GraphQL schema file: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +dast_api: + variables: + DAST_API_GRAPHQL: /api/graphql + DAST_API_GRAPHQL_SCHEMA: test-api-graphql.schema + DAST_API_TARGET_URL: http://test-deployment/ +``` + +Complete example configuration of using an GraphQL schema file URL: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +dast_api: + variables: + DAST_API_GRAPHQL: /api/graphql + DAST_API_GRAPHQL_SCHEMA: http://file-store/files/test-api-graphql.schema + DAST_API_TARGET_URL: http://test-deployment/ +``` + +This example is a minimal configuration for API security testing. From here you can: + +- [Run your first scan](#running-your-first-scan). +- [Add authentication](customizing_analyzer_settings.md#authentication). +- Learn how to [handle false positives](#handling-false-positives). + +## Postman Collection + +The [Postman API Client](https://www.postman.com/product/api-client/) is a popular tool that +developers and testers use to call various types of APIs. The API definitions +[can be exported as a Postman Collection file](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-postman-data) +for use with API security testing. When exporting, make sure to select a supported version of Postman +Collection: v2.0 or v2.1. + +When used with the GitLab API security testing scanner, Postman Collections must contain definitions of the web API to +test with valid data. The API security testing scanner extracts all the API definitions and uses them to perform +testing. + +WARNING: +Postman Collection files may contain sensitive information such as authentication tokens, API keys, +and session cookies. We recommend that you review the Postman Collection file contents before adding +them to a repository. + +### API security testing scanning with a Postman Collection file + +To configure API security testing to use a Postman Collection file that provides information about the target +API to test: + +1. [Include](../../../../ci/yaml/index.md#includetemplate) + the [`DAST-API.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml). + +1. The [configuration file](variables.md#configuration-files) has several testing profiles defined with different checks enabled. We recommend that you start with the `Quick` profile. + Testing with this profile completes faster, allowing for easier configuration validation. + + Provide the profile by adding the `DAST_API_PROFILE` CI/CD variable to your `.gitlab-ci.yml` file. + +1. Provide the location of the Postman Collection file as either a file or URL. Specify the location by adding the `DAST_API_POSTMAN_COLLECTION` variable. + +1. The target API instance's base URL is also required. Provide it by using the `DAST_API_TARGET_URL` + variable or an `environment_url.txt` file. + + Adding the URL in an `environment_url.txt` file at your project's root is great for testing in + dynamic environments. To run API security testing against an app dynamically created during a GitLab CI/CD + pipeline, have the app persist its URL in an `environment_url.txt` file. API security testing + automatically parses that file to find its scan target. You can see an + [example of this in our Auto DevOps CI YAML](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml). + +Complete example configuration of using a Postman collection: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_POSTMAN_COLLECTION: postman-collection_serviceA.json + DAST_API_TARGET_URL: http://test-deployment/ +``` + +This is a minimal configuration for API security testing. From here you can: + +- [Run your first scan](#running-your-first-scan). +- [Add authentication](customizing_analyzer_settings.md#authentication). +- Learn how to [handle false positives](#handling-false-positives). + +### Postman variables + +> - Support for Postman Environment file format was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356312) in GitLab 15.1. +> - Support for multiple variable files was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356312) in GitLab 15.1. +> - Support for Postman variable scopes: Global and Environment was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356312) in GitLab 15.1. + +#### Variables in Postman Client + +Postman allows the developer to define placeholders that can be used in different parts of the +requests. These placeholders are called variables, as explained in [using variables](https://learning.postman.com/docs/sending-requests/variables/). +You can use variables to store and reuse values in your requests and scripts. For example, you can +edit the collection to add variables to the document: + +![Edit collection variable tab View](../img/dast_api_postman_collection_edit_variable.png) + +Or alternatively, you can add variables in an environment: + +![Edit environment variables View](../img/dast_api_postman_environment_edit_variable.png) + +You can then use the variables in sections such as URL, headers, and others: + +![Edit request using variables View](../img/dast_api_postman_request_edit.png) + +Postman has grown from a basic client tool with a nice UX experience to a more complex ecosystem that allows testing APIs with scripts, creating complex collections that trigger secondary requests, and setting variables along the way. Not every feature in the Postman ecosystem is supported. For example, scripts are not supported. The main focus of the Postman support is to ingest Postman Collection definitions that are used by the Postman Client and their related variables defined in the workspace, environments, and the collections themselves. + +Postman allows creating variables in different scopes. Each scope has a different level of visibility in the Postman tools. For example, you can create a variable in a _global environment_ scope that is seen by every operation definition and workspace. You can also create a variable in a specific _environment_ scope that is only visible and used when that specific environment is selected for use. Some scopes are not always available, for example in the Postman ecosystem you can create requests in the Postman Client, these requests do not have a _local_ scope, but test scripts do. + +Variable scopes in Postman can be a daunting topic and not everyone is familiar with it. We strongly recommend that you read [Variable Scopes](https://learning.postman.com/docs/sending-requests/variables/#variable-scopes) from Postman documentation before moving forward. + +As mentioned above, there are different variable scopes, and each of them has a purpose and can be used to provide more flexibility to your Postman document. There is an important note on how values for variables are computed, as per Postman documentation: + +> If a variable with the same name is declared in two different scopes, the value stored in the variable with narrowest scope is used. For example, if there is a global variable named `username` and a local variable named `username`, the local value is used when the request runs. + +The following is a summary of the variable scopes supported by the Postman Client and API security testing: + +- **Global Environment (Global) scope** is a special pre-defined environment that is available throughout a workspace. We can also refer to the _global environment_ scope as the _global_ scope. The Postman Client allows exporting the global environment into a JSON file, which can be used with API security testing. +- **Environment scope** is a named group of variables created by a user in the Postman Client. + The Postman Client supports a single active environment along with the global environment. The variables defined in an active user-created environment take precedence over variables defined in the global environment. The Postman Client allows exporting your environment into a JSON file, which can be used with API security testing. +- **Collection scope** is a group of variables declared in a given collection. The collection variables are available to the collection where they have been declared and the nested requests or collections. Variables defined in the collection scope take precedence over the _global environment_ scope and also the _environment_ scope. + The Postman Client can export one or more collections into a JSON file, this JSON file contains selected collections, requests, and collection variables. +- **API security testing scope** is a new scope added by API security testing to allow users to provide extra variables, or override variables defined in other supported scopes. This scope is not supported by Postman. The _API security testing scope_ variables are provided using a [custom JSON file format](#api-security-testing-scope-custom-json-file-format). + - Override values defined in the environment or collection + - Defining variables from scripts + - Define a single row of data from the unsupported _data scope_ +- **Data scope** is a group of variables in which their name and values come from JSON or CSV files. A Postman collection runner like [Newman](https://learning.postman.com/docs/running-collections/using-newman-cli/command-line-integration-with-newman/) or [Postman Collection Runner](https://learning.postman.com/docs/running-collections/intro-to-collection-runs/) executes the requests in a collection as many times as entries have the JSON or CSV file. A good use case for these variables is to automate tests using scripts in Postman. + API security testing does **not** support reading data from a CSV or JSON file. +- **Local scope** are variables that are defined in Postman scripts. API security testing does **not** support Postman scripts and by extension, variables defined in scripts. You can still provide values for the script-defined variables by defining them in one of the supported scopes, or our custom JSON format. + +Not all scopes are supported by API security testing and variables defined in scripts are not supported. The following table is sorted by broadest scope to narrowest scope. + +| Scope | Postman | API security testing | Comment | +|----------------------------|:-------:|:--------------------:|:-------------------------------------------| +| Global Environment | Yes | Yes | Special pre-defined environment | +| Environment | Yes | Yes | Named environments | +| Collection | Yes | Yes | Defined in your postman collection | +| API security testing scope | No | Yes | Custom scope added by API security testing | +| Data | Yes | No | External files in CSV or JSON format | +| Local | Yes | No | Variables defined in scripts | + +For more details on how to define variables and export variables in different scopes, see: + +- [Defining collection variables](https://learning.postman.com/docs/sending-requests/variables/#defining-collection-variables) +- [Defining environment variables](https://learning.postman.com/docs/sending-requests/variables/#defining-environment-variables) +- [Defining global variables](https://learning.postman.com/docs/sending-requests/variables/#defining-global-variables) + +##### Exporting from Postman Client + +The Postman Client lets you export different file formats, for instance, you can export a Postman collection or a Postman environment. +The exported environment can be the global environment (which is always available) or can be any custom environment you previously have created. When you export a Postman Collection, it may contain only declarations for _collection_ and _local_ scoped variables; _environment_ scoped variables are not included. + +To get the declaration for _environment_ scoped variables, you have to export a given environment at the time. Each exported file only includes variables from the selected environment. + +For more details on exporting variables in different supported scopes, see: + +- [Exporting collections](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-collections) +- [Exporting environments](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-environments) +- [Downloading global environments](https://learning.postman.com/docs/sending-requests/variables/#downloading-global-environments) + +#### API security testing scope, custom JSON file format + +Our custom JSON file format is a JSON object where each object property represents a variable name and the property value represents the variable value. This file can be created using your favorite text editor, or it can be produced by an earlier job in your pipeline. + +This example defines two variables `base_url` and `token` in the API security testing scope: + +```json +{ + "base_url": "http://127.0.0.1/", + "token": "Token 84816165151" +} +``` + +#### Using scopes with API security testing + +The scopes: _global_, _environment_, _collection_, and _GitLab API security testing_ are supported in [GitLab 15.1 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/356312). GitLab 15.0 and earlier, supports only the _collection_, and _GitLab API security testing_ scopes. + +The following table provides a quick reference for mapping scope files/URLs to API security testing configuration variables: + +| Scope | How to Provide | +| ------------------ | --------------- | +| Global environment | DAST_API_POSTMAN_COLLECTION_VARIABLES | +| Environment | DAST_API_POSTMAN_COLLECTION_VARIABLES | +| Collection | DAST_API_POSTMAN_COLLECTION | +| API security testing scope | DAST_API_POSTMAN_COLLECTION_VARIABLES | +| Data | Not supported | +| Local | Not supported | + +The Postman Collection document automatically includes any _collection_ scoped variables. The Postman Collection is provided with the configuration variable `DAST_API_POSTMAN_COLLECTION`. This variable can be set to a single [exported Postman collection](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-collections). + +Variables from other scopes are provided through the `DAST_API_POSTMAN_COLLECTION_VARIABLES` configuration variable. The configuration variable supports a comma (`,`) delimited file list in [GitLab 15.1 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/356312). GitLab 15.0 and earlier, supports only one single file. The order of the files provided is not important as the files provide the needed scope information. + +The configuration variable `DAST_API_POSTMAN_COLLECTION_VARIABLES` can be set to: + +- [Exported Global environment](https://learning.postman.com/docs/sending-requests/variables/#downloading-global-environments) +- [Exported environments](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-environments) +- [API security testing Custom JSON format](#api-security-testing-scope-custom-json-file-format) + +#### Undefined Postman variables + +There is a chance that API security testing engine does not find all variables references that your Postman collection file is using. Some cases can be: + +- You are using _data_ or _local_ scoped variables, and as stated previously these scopes are not supported by API security testing. Thus, assuming the values for these variables have not been provided through [the API security testing scope](#api-security-testing-scope-custom-json-file-format), then the values of the _data_ and _local_ scoped variables are undefined. +- A variable name was typed incorrectly, and the name does not match the defined variable. +- Postman Client supports a new dynamic variable that is not supported by API security testing. + +When possible, API security testing follows the same behavior as the Postman Client does when dealing with undefined variables. The text of the variable reference remains the same, and there is no text substitution. The same behavior also applies to any unsupported dynamic variables. + +For example, if a request definition in the Postman Collection references the variable `{{full_url}}` and the variable is not found it is left unchanged with the value `{{full_url}}`. + +#### Dynamic Postman variables + +In addition to variables that a user can define at various scope levels, Postman has a set of pre-defined variables called _dynamic_ variables. The [_dynamic_ variables](https://learning.postman.com/docs/writing-scripts/script-references/variables-list/) are already defined and their name is prefixed with a dollar sign (`$`), for instance, `$guid`. _Dynamic_ variables can be used like any other variable, and in the Postman Client, they produce random values during the request/collection run. + +An important difference between API security testing and Postman is that API security testing returns the same value for each usage of the same dynamic variables. This differs from the Postman Client behavior which returns a random value on each use of the same dynamic variable. In other words, API security testing uses static values for dynamic variables while Postman uses random values. + +The supported dynamic variables during the scanning process are: + +| Variable | Value | +| ----------- | ----------- | +| `$guid` | `611c2e81-2ccb-42d8-9ddc-2d0bfa65c1b4` | +| `$isoTimestamp` | `2020-06-09T21:10:36.177Z` | +| `$randomAbbreviation` | `PCI` | +| `$randomAbstractImage` | `http://no-a-valid-host/640/480/abstract` | +| `$randomAdjective` | `auxiliary` | +| `$randomAlphaNumeric` | `a` | +| `$randomAnimalsImage` | `http://no-a-valid-host/640/480/animals` | +| `$randomAvatarImage` | `https://no-a-valid-host/path/to/some/image.jpg` | +| `$randomBankAccount` | `09454073` | +| `$randomBankAccountBic` | `EZIAUGJ1` | +| `$randomBankAccountIban` | `MU20ZPUN3039684000618086155TKZ` | +| `$randomBankAccountName` | `Home Loan Account` | +| `$randomBitcoin` | `3VB8JGT7Y4Z63U68KGGKDXMLLH5` | +| `$randomBoolean` | `true` | +| `$randomBs` | `killer leverage schemas` | +| `$randomBsAdjective` | `viral` | +| `$randomBsBuzz` | `repurpose` | +| `$randomBsNoun` | `markets` | +| `$randomBusinessImage` | `http://no-a-valid-host/640/480/business` | +| `$randomCatchPhrase` | `Future-proofed heuristic open architecture` | +| `$randomCatchPhraseAdjective` | `Business-focused` | +| `$randomCatchPhraseDescriptor` | `bandwidth-monitored` | +| `$randomCatchPhraseNoun` | `superstructure` | +| `$randomCatsImage` | `http://no-a-valid-host/640/480/cats` | +| `$randomCity` | `Spinkahaven` | +| `$randomCityImage` | `http://no-a-valid-host/640/480/city` | +| `$randomColor` | `fuchsia` | +| `$randomCommonFileExt` | `wav` | +| `$randomCommonFileName` | `well_modulated.mpg4` | +| `$randomCommonFileType` | `audio` | +| `$randomCompanyName` | `Grady LLC` | +| `$randomCompanySuffix` | `Inc` | +| `$randomCountry` | `Kazakhstan` | +| `$randomCountryCode` | `MD` | +| `$randomCreditCardMask` | `3622` | +| `$randomCurrencyCode` | `ZMK` | +| `$randomCurrencyName` | `Pound Sterling` | +| `$randomCurrencySymbol` | `£` | +| `$randomDatabaseCollation` | `utf8_general_ci` | +| `$randomDatabaseColumn` | `updatedAt` | +| `$randomDatabaseEngine` | `Memory` | +| `$randomDatabaseType` | `text` | +| `$randomDateFuture` | `Tue Mar 17 2020 13:11:50 GMT+0530 (India Standard Time)` | +| `$randomDatePast` | `Sat Mar 02 2019 09:09:26 GMT+0530 (India Standard Time)` | +| `$randomDateRecent` | `Tue Jul 09 2019 23:12:37 GMT+0530 (India Standard Time)` | +| `$randomDepartment` | `Electronics` | +| `$randomDirectoryPath` | `/usr/local/bin` | +| `$randomDomainName` | `trevor.info` | +| `$randomDomainSuffix` | `org` | +| `$randomDomainWord` | `jaden` | +| `$randomEmail` | `Iva.Kovacek61@no-a-valid-host.com` | +| `$randomExampleEmail` | `non-a-valid-user@example.net` | +| `$randomFashionImage` | `http://no-a-valid-host/640/480/fashion` | +| `$randomFileExt` | `war` | +| `$randomFileName` | `neural_sri_lanka_rupee_gloves.gdoc` | +| `$randomFilePath` | `/home/programming_chicken.cpio` | +| `$randomFileType` | `application` | +| `$randomFirstName` | `Chandler` | +| `$randomFoodImage` | `http://no-a-valid-host/640/480/food` | +| `$randomFullName` | `Connie Runolfsdottir` | +| `$randomHexColor` | `#47594a` | +| `$randomImageDataUri` | `data:image/svg+xml;charset=UTF-8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20version%3D%221.1%22%20baseProfile%3D%22full%22%20width%3D%22undefined%22%20height%3D%22undefined%22%3E%20%3Crect%20width%3D%22100%25%22%20height%3D%22100%25%22%20fill%3D%22grey%22%2F%3E%20%20%3Ctext%20x%3D%220%22%20y%3D%2220%22%20font-size%3D%2220%22%20text-anchor%3D%22start%22%20fill%3D%22white%22%3Eundefinedxundefined%3C%2Ftext%3E%20%3C%2Fsvg%3E` | +| `$randomImageUrl` | `http://no-a-valid-host/640/480` | +| `$randomIngverb` | `navigating` | +| `$randomInt` | `494` | +| `$randomIP` | `241.102.234.100` | +| `$randomIPV6` | `dbe2:7ae6:119b:c161:1560:6dda:3a9b:90a9` | +| `$randomJobArea` | `Mobility` | +| `$randomJobDescriptor` | `Senior` | +| `$randomJobTitle` | `International Creative Liaison` | +| `$randomJobType` | `Supervisor` | +| `$randomLastName` | `Schneider` | +| `$randomLatitude` | `55.2099` | +| `$randomLocale` | `ny` | +| `$randomLongitude` | `40.6609` | +| `$randomLoremLines` | `Ducimus in ut mollitia.\nA itaque non.\nHarum temporibus nihil voluptas.\nIste in sed et nesciunt in quaerat sed.` | +| `$randomLoremParagraph` | `Ab aliquid odio iste quo voluptas voluptatem dignissimos velit. Recusandae facilis qui commodi ea magnam enim nostrum quia quis. Nihil est suscipit assumenda ut voluptatem sed. Esse ab voluptas odit qui molestiae. Rem est nesciunt est quis ipsam expedita consequuntur.` | +| `$randomLoremParagraphs` | `Voluptatem rem magnam aliquam ab id aut quaerat. Placeat provident possimus voluptatibus dicta velit non aut quasi. Mollitia et aliquam expedita sunt dolores nam consequuntur. Nam dolorum delectus ipsam repudiandae et ipsam ut voluptatum totam. Nobis labore labore recusandae ipsam quo.` | +| `$randomLoremSentence` | `Molestias consequuntur nisi non quod.` | +| `$randomLoremSentences` | `Et sint voluptas similique iure amet perspiciatis vero sequi atque. Ut porro sit et hic. Neque aspernatur vitae fugiat ut dolore et veritatis. Ab iusto ex delectus animi. Voluptates nisi iusto. Impedit quod quae voluptate qui.` | +| `$randomLoremSlug` | `eos-aperiam-accusamus, beatae-id-molestiae, qui-est-repellat` | +| `$randomLoremText` | `Quisquam asperiores exercitationem ut ipsum. Aut eius nesciunt. Et reiciendis aut alias eaque. Nihil amet laboriosam pariatur eligendi. Sunt ullam ut sint natus ducimus. Voluptas harum aspernatur soluta rem nam.` | +| `$randomLoremWord` | `est` | +| `$randomLoremWords` | `vel repellat nobis` | +| `$randomMACAddress` | `33:d4:68:5f:b4:c7` | +| `$randomMimeType` | `audio/vnd.vmx.cvsd` | +| `$randomMonth` | `February` | +| `$randomNamePrefix` | `Dr.` | +| `$randomNameSuffix` | `MD` | +| `$randomNatureImage` | `http://no-a-valid-host/640/480/nature` | +| `$randomNightlifeImage` | `http://no-a-valid-host/640/480/nightlife` | +| `$randomNoun` | `bus` | +| `$randomPassword` | `t9iXe7COoDKv8k3` | +| `$randomPeopleImage` | `http://no-a-valid-host/640/480/people` | +| `$randomPhoneNumber` | `700-008-5275` | +| `$randomPhoneNumberExt` | `27-199-983-3864` | +| `$randomPhrase` | `You can't program the monitor without navigating the mobile XML program!` | +| `$randomPrice` | `531.55` | +| `$randomProduct` | `Pizza` | +| `$randomProductAdjective` | `Unbranded` | +| `$randomProductMaterial` | `Steel` | +| `$randomProductName` | `Handmade Concrete Tuna` | +| `$randomProtocol` | `https` | +| `$randomSemver` | `7.0.5` | +| `$randomSportsImage` | `http://no-a-valid-host/640/480/sports` | +| `$randomStreetAddress` | `5742 Harvey Streets` | +| `$randomStreetName` | `Kuhic Island` | +| `$randomTransactionType` | `payment` | +| `$randomTransportImage` | `http://no-a-valid-host/640/480/transport` | +| `$randomUrl` | `https://no-a-valid-host.net` | +| `$randomUserAgent` | `Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.9.8; rv:15.6) Gecko/20100101 Firefox/15.6.6` | +| `$randomUserName` | `Jarrell.Gutkowski` | +| `$randomUUID` | `6929bb52-3ab2-448a-9796-d6480ecad36b` | +| `$randomVerb` | `navigate` | +| `$randomWeekday` | `Thursday` | +| `$randomWord` | `withdrawal` | +| `$randomWords` | `Samoa Synergistic sticky copying Grocery` | +| `$timestamp` | `1562757107` | + +#### Example: Global Scope + +In this example, [the _global_ scope is exported](https://learning.postman.com/docs/sending-requests/variables/#downloading-global-environments) from the Postman Client as `global-scope.json` and provided to API security testing through the `DAST_API_POSTMAN_COLLECTION_VARIABLES` configuration variable. + +Here is an example of using `DAST_API_POSTMAN_COLLECTION_VARIABLES`: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_POSTMAN_COLLECTION: postman-collection.json + DAST_API_POSTMAN_COLLECTION_VARIABLES: global-scope.json + DAST_API_TARGET_URL: http://test-deployment/ +``` + +#### Example: Environment Scope + +In this example, [the _environment_ scope is exported](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-environments) from the Postman Client as `environment-scope.json` and provided to API security testing through the `DAST_API_POSTMAN_COLLECTION_VARIABLES` configuration variable. + +Here is an example of using `DAST_API_POSTMAN_COLLECTION_VARIABLES`: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_POSTMAN_COLLECTION: postman-collection.json + DAST_API_POSTMAN_COLLECTION_VARIABLES: environment-scope.json + DAST_API_TARGET_URL: http://test-deployment/ +``` + +#### Example: Collection Scope + +The _collection_ scope variables are included in the exported Postman Collection file and provided through the `DAST_API_POSTMAN_COLLECTION` configuration variable. + +Here is an example of using `DAST_API_POSTMAN_COLLECTION`: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_POSTMAN_COLLECTION: postman-collection.json + DAST_API_TARGET_URL: http://test-deployment/ +``` + +#### Example: API security testing scope + +The API security testing scope is used for two main purposes, defining _data_ and _local_ scope variables that are not supported by API security testing, and changing the value of an existing variable defined in another scope. The API security testing scope is provided through the `DAST_API_POSTMAN_COLLECTION_VARIABLES` configuration variable. + +Here is an example of using `DAST_API_POSTMAN_COLLECTION_VARIABLES`: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_POSTMAN_COLLECTION: postman-collection.json + DAST_API_POSTMAN_COLLECTION_VARIABLES: dast-api-scope.json + DAST_API_TARGET_URL: http://test-deployment/ +``` + +The file `dast-api-scope.json` uses our [custom JSON file format](#api-security-testing-scope-custom-json-file-format). This JSON is an object with key-value pairs for properties. The keys are the variables' names, and the values are the variables' +values. For example: + +```json +{ + "base_url": "http://127.0.0.1/", + "token": "Token 84816165151" +} +``` + +#### Example: Multiple Scopes + +In this example, a _global_ scope, _environment_ scope, and _collection_ scope are configured. The first step is to export our various scopes. + +- [Export the _global_ scope](https://learning.postman.com/docs/sending-requests/variables/#downloading-global-environments) as `global-scope.json` +- [Export the _environment_ scope](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-environments) as `environment-scope.json` +- Export the Postman Collection which includes the _collection_ scope as `postman-collection.json` + +The Postman Collection is provided using the `DAST_API_POSTMAN_COLLECTION` variable, while the other scopes are provided using the `DAST_API_POSTMAN_COLLECTION_VARIABLES`. API security testing can identify which scope the provided files match using data provided in each file. + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_POSTMAN_COLLECTION: postman-collection.json + DAST_API_POSTMAN_COLLECTION_VARIABLES: global-scope.json,environment-scope.json + DAST_API_TARGET_URL: http://test-deployment/ +``` + +#### Example: Changing a Variables Value + +When using exported scopes, it's often the case that the value of a variable must be changed for use with API security testing. For example, a _collection_ scoped variable might contain a variable named `api_version` with a value of `v2`, while your test needs a value of `v1`. Instead of modifying the exported collection to change the value, the API security testing scope can be used to change its value. This works because the _API security testing_ scope takes precedence over all other scopes. + +The _collection_ scope variables are included in the exported Postman Collection file and provided through the `DAST_API_POSTMAN_COLLECTION` configuration variable. + +The API security testing scope is provided through the `DAST_API_POSTMAN_COLLECTION_VARIABLES` configuration variable, but first, we must create the file. +The file `dast-api-scope.json` uses our [custom JSON file format](#api-security-testing-scope-custom-json-file-format). This JSON is an object with key-value pairs for properties. The keys are the variables' names, and the values are the variables' +values. For example: + +```json +{ + "api_version": "v1" +} +``` + +Our CI definition: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_POSTMAN_COLLECTION: postman-collection.json + DAST_API_POSTMAN_COLLECTION_VARIABLES: dast-api-scope.json + DAST_API_TARGET_URL: http://test-deployment/ +``` + +#### Example: Changing a Variables Value with Multiple Scopes + +When using exported scopes, it's often the case that the value of a variable must be changed for use with API security testing. For example, an _environment_ scope might contain a variable named `api_version` with a value of `v2`, while your test needs a value of `v1`. Instead of modifying the exported file to change the value, the API security testing scope can be used. This works because the _API security testing_ scope takes precedence over all other scopes. + +In this example, a _global_ scope, _environment_ scope, _collection_ scope, and _API security testing_ scope are configured. The first step is to export and create our various scopes. + +- [Export the _global_ scope](https://learning.postman.com/docs/sending-requests/variables/#downloading-global-environments) as `global-scope.json` +- [Export the _environment_ scope](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-environments) as `environment-scope.json` +- Export the Postman Collection which includes the _collection_ scope as `postman-collection.json` + +The API security testing scope is used by creating a file `dast-api-scope.json` using our [custom JSON file format](#api-security-testing-scope-custom-json-file-format). This JSON is an object with key-value pairs for properties. The keys are the variables' names, and the values are the variables' +values. For example: + +```json +{ + "api_version": "v1" +} +``` + +The Postman Collection is provided using the `DAST_API_POSTMAN_COLLECTION` variable, while the other scopes are provided using the `DAST_API_POSTMAN_COLLECTION_VARIABLES`. API security testing can identify which scope the provided files match using data provided in each file. + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_POSTMAN_COLLECTION: postman-collection.json + DAST_API_POSTMAN_COLLECTION_VARIABLES: global-scope.json,environment-scope.json,dast-api-scope.json + DAST_API_TARGET_URL: http://test-deployment/ +``` + +## Running your first scan + +When configured correctly, a CI/CD pipeline contains a `dast` stage and an `dast_api` job. The job only fails when an invalid configuration is provided. During typical operation, the job always succeeds even if vulnerabilities are identified during testing. + +Vulnerabilities are displayed on the **Security** pipeline tab with the suite name. When testing against the repositories default branch, the API security testing vulnerabilities are also shown on the Security and Compliance's Vulnerability Report page. + +To prevent an excessive number of reported vulnerabilities, the API security testing scanner limits the number of vulnerabilities it reports per operation. + +## Viewing API security testing vulnerabilities + +The API security testing analyzer produces a JSON report that is collected and used +[to populate the vulnerabilities into GitLab vulnerability screens](#view-details-of-an-api-security-testing-vulnerability). + +See [handling false positives](#handling-false-positives) for information about configuration changes you can make to limit the number of false positives reported. + +### View details of an API security testing vulnerability + +Follow these steps to view details of a vulnerability: + +1. You can view vulnerabilities in a project, or a merge request: + + - In a project, go to the project's **Secure > Vulnerability report** + page. This page shows all vulnerabilities from the default branch only. + - In a merge request, go the merge request's **Security** section and select the **Expand** + button. API security testing vulnerabilities are available in a section labeled + **DAST detected N potential vulnerabilities**. Select the title to display the vulnerability + details. + +1. Select the vulnerabilities title to display the details. The table below describes these details. + + | Field | Description | + |:--------------------|:----------------------------------------------------------------------------------------| + | Description | Description of the vulnerability including what was modified. | + | Project | Namespace and project in which the vulnerability was detected. | + | Method | HTTP method used to detect the vulnerability. | + | URL | URL at which the vulnerability was detected. | + | Request | The HTTP request that caused the vulnerability. | + | Unmodified Response | Response from an unmodified request. This is what a typical working response looks like.| + | Actual Response | Response received from test request. | + | Evidence | How we determined a vulnerability occurred. | + | Identifiers | The API security testing check used to find this vulnerability. | + | Severity | Severity of the vulnerability. | + | Scanner Type | Scanner used to perform testing. | + +### Security Dashboard + +The Security Dashboard is a good place to get an overview of all the security vulnerabilities in your groups, projects and +pipelines. For more information, see the [Security Dashboard documentation](../../security_dashboard/index.md). + +### Interacting with the vulnerabilities + +Once a vulnerability is found, you can interact with it. Read more on how to +[address the vulnerabilities](../../vulnerabilities/index.md). + +### Handling False Positives + +False positives can be handled in several ways: + +- Dismiss the vulnerability. +- Some checks have several methods of detecting when a vulnerability is identified, called _Assertions_. + Assertions can also be turned off and configured. For example, the API security testing scanner by default uses HTTP + status codes to help identify when something is a real issue. If an API returns a 500 error during + testing, this creates a vulnerability. This isn't always desired, as some frameworks return 500 errors often. +- Turn off the Check producing the false positive. This prevents the check from generating any + vulnerabilities. Example checks are the SQL Injection Check, and JSON Hijacking Check. + +#### Turn off a Check + +Checks perform testing of a specific type and can be turned on and off for specific configuration +profiles. The provided [configuration files](variables.md#configuration-files) define several profiles that you +can use. The profile definition in the configuration file lists all the checks that are active +during a scan. To turn off a specific check, remove it from the profile definition in the +configuration file. The profiles are defined in the `Profiles` section of the configuration file. + +Example profile definition: + +```yaml +Profiles: + - Name: Quick + DefaultProfile: Empty + Routes: + - Route: *Route0 + Checks: + - Name: ApplicationInformationCheck + - Name: CleartextAuthenticationCheck + - Name: FrameworkDebugModeCheck + - Name: HtmlInjectionCheck + - Name: InsecureHttpMethodsCheck + - Name: JsonHijackingCheck + - Name: JsonInjectionCheck + - Name: SensitiveInformationCheck + - Name: SessionCookieCheck + - Name: SqlInjectionCheck + - Name: TokenCheck + - Name: XmlInjectionCheck +``` + +To turn off the JSON Hijacking Check you can remove these lines: + +```yaml + - Name: JsonHijackingCheck +``` + +This results in the following YAML: + +```yaml +- Name: Quick + DefaultProfile: Empty + Routes: + - Route: *Route0 + Checks: + - Name: ApplicationInformationCheck + - Name: CleartextAuthenticationCheck + - Name: FrameworkDebugModeCheck + - Name: HtmlInjectionCheck + - Name: InsecureHttpMethodsCheck + - Name: JsonInjectionCheck + - Name: SensitiveInformationCheck + - Name: SessionCookieCheck + - Name: SqlInjectionCheck + - Name: TokenCheck + - Name: XmlInjectionCheck +``` + +#### Turn off an Assertion for a Check + +Assertions detect vulnerabilities in tests produced by checks. Many checks support multiple Assertions such as Log Analysis, Response Analysis, and Status Code. When a vulnerability is found, the Assertion used is provided. To identify which Assertions are on by default, see the Checks default configuration in the configuration file. The section is called `Checks`. + +This example shows the SQL Injection Check: + +```yaml +- Name: SqlInjectionCheck + Configuration: + UserInjections: [] + Assertions: + - Name: LogAnalysisAssertion + - Name: ResponseAnalysisAssertion + - Name: StatusCodeAssertion +``` + +Here you can see three Assertions are on by default. A common source of false positives is +`StatusCodeAssertion`. To turn it off, modify its configuration in the `Profiles` section. This +example provides only the other two Assertions (`LogAnalysisAssertion`, +`ResponseAnalysisAssertion`). This prevents `SqlInjectionCheck` from using `StatusCodeAssertion`: + +```yaml +Profiles: + - Name: Quick + DefaultProfile: Empty + Routes: + - Route: *Route0 + Checks: + - Name: ApplicationInformationCheck + - Name: CleartextAuthenticationCheck + - Name: FrameworkDebugModeCheck + - Name: HtmlInjectionCheck + - Name: InsecureHttpMethodsCheck + - Name: JsonHijackingCheck + - Name: JsonInjectionCheck + - Name: SensitiveInformationCheck + - Name: SessionCookieCheck + - Name: SqlInjectionCheck + Assertions: + - Name: LogAnalysisAssertion + - Name: ResponseAnalysisAssertion + - Name: TokenCheck + - Name: XmlInjectionCheck +``` diff --git a/doc/user/application_security/api_security_testing/configuration/index.md b/doc/user/application_security/api_security_testing/configuration/index.md new file mode 100644 index 00000000000..189ef14929c --- /dev/null +++ b/doc/user/application_security/api_security_testing/configuration/index.md @@ -0,0 +1,15 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: reference, howto +--- + +# Configuration + +- [Requirements](requirements.md) +- [Enabling the analyzer](enabling_the_analyzer.md) +- [Customize analyzer settings](customizing_analyzer_settings.md) +- [Overriding analyzer jobs](overriding_analyzer_jobs.md) +- [Available CI/CD variables](variables.md) +- [Offline configuration](offline_configuration.md) diff --git a/doc/user/application_security/api_security_testing/configuration/offline_configuration.md b/doc/user/application_security/api_security_testing/configuration/offline_configuration.md new file mode 100644 index 00000000000..9bb96bfc384 --- /dev/null +++ b/doc/user/application_security/api_security_testing/configuration/offline_configuration.md @@ -0,0 +1,30 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Offline configuration + +For self-managed GitLab instances in an environment with limited, restricted, or intermittent access to external resources through the internet, some adjustments are required for the API security testing job to successfully run. + +Steps: + +1. Host the Docker image in a local container registry. +1. Set the `SECURE_ANALYZERS_PREFIX` to the local container registry. + +The Docker image for API security testing must be pulled (downloaded) from the public registry and then pushed (imported) into a local registry. The GitLab container registry can be used to locally host the Docker image. This process can be performed using a special template. See [loading Docker images onto your offline host](../../offline_deployments/index.md#loading-docker-images-onto-your-offline-host) for instructions. + +Once the Docker image is hosted locally, the `SECURE_ANALYZERS_PREFIX` variable is set with the location of the local registry. The variable must be set such that concatenating `/api-security:2` results in a valid image location. + +NOTE: +API security testing and API Fuzzing both use the same underlying Docker image `api-security:2`. + +For example, the below line sets a registry for the image `registry.gitlab.com/security-products/api-security:2`: + +`SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products"` + +NOTE: +Setting `SECURE_ANALYZERS_PREFIX` changes the Docker image registry location for all GitLab Secure templates. + +For more information, see [Offline environments](../../offline_deployments/index.md). diff --git a/doc/user/application_security/api_security_testing/configuration/overriding_analyzer_jobs.md b/doc/user/application_security/api_security_testing/configuration/overriding_analyzer_jobs.md new file mode 100644 index 00000000000..99ca752c923 --- /dev/null +++ b/doc/user/application_security/api_security_testing/configuration/overriding_analyzer_jobs.md @@ -0,0 +1,21 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: reference, howto +--- + +# Overriding API security testing jobs + +To override a job definition, (for example, change properties like `variables`, `dependencies`, or [`rules`](../../../../ci/yaml/index.md#rules)), +declare a job with the same name as the DAST job to override. Place this new job after the template +inclusion and specify any additional keys under it. For example, this sets the target APIs base URL: + +```yaml +include: + - template: Security/DAST-API.gitlab-ci.yml + +dast_api: + variables: + DAST_API_TARGET_URL: https://target/api +``` diff --git a/doc/user/application_security/api_security_testing/configuration/requirements.md b/doc/user/application_security/api_security_testing/configuration/requirements.md new file mode 100644 index 00000000000..a680e8c143c --- /dev/null +++ b/doc/user/application_security/api_security_testing/configuration/requirements.md @@ -0,0 +1,114 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: reference, howto +--- + +# Requirements + +- A web API using one of the supported API types: + - REST API + - SOAP + - GraphQL + - Form bodies, JSON, or XML +- An API specification in one of the following formats: + - [OpenAPI v2 or v3 Specification](enabling_the_analyzer.md#openapi-specification) + - [GraphQL Schema](enabling_the_analyzer.md#graphql-schema) + - [HTTP Archive (HAR)](enabling_the_analyzer.md#http-archive-har) + - [Postman Collection v2.0 or v2.1](enabling_the_analyzer.md#postman-collection) +- [GitLab Runner](../../../../ci/runners/index.md) available, with the + [`docker` executor](https://docs.gitlab.com/runner/executors/docker.html) on Linux/amd64. +- Target application deployed. For more details, read [Deployment options](#application-deployment-options). +- `dast` stage added to the CI/CD pipeline definition. This should be added after the deploy step, for example: + + ```yaml + stages: + - build + - test + - deploy + - dast + ``` + +## Recommendations + +- Configure runners to use the [always pull policy](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy) to run the latest versions of the analyzers. +- By default, API security testing downloads all artifacts defined by previous jobs in the pipeline. If + your DAST job does not rely on `environment_url.txt` to define the URL under test or any other files created + in previous jobs, we recommend you don't download artifacts. To avoid downloading + artifacts, extend the analyzer CI/CD job to specify no dependencies. For example, for the DAST proxy-based analyzer add the following to your `.gitlab-ci.yml` file: + + ```yaml + dast_api: + dependencies: [] + ``` + +## Application deployment options + +API security testing requires a deployed application to be available to scan. + +Depending on the complexity of the target application, there are a few options as to how to deploy and configure +the API security testing template. + +### Review Apps + +Review Apps are the most involved method of deploying your DAST target application. To assist in the process, +we created a Review App deployment using Google Kubernetes Engine (GKE). This example can be found in our +[Review Apps - GKE](https://gitlab.com/gitlab-org/security-products/demos/dast/review-app-gke) project, along with detailed +instructions in the [README.md](https://gitlab.com/gitlab-org/security-products/demos/dast/review-app-gke/-/blob/master/README.md) +on how to configure Review Apps for DAST. + +### Docker Services + +If your application uses Docker containers you have another option for deploying and scanning with DAST. +After your Docker build job completes and your image is added to your container registry, you can use the image as a +[service](../../../../ci/services/index.md). + +By using service definitions in your `.gitlab-ci.yml`, you can scan services with the DAST analyzer. + +When adding a `services` section to the job, the `alias` is used to define the hostname that can be used to access the service. In the following example, the `alias: yourapp` portion of the `dast` job definition means that the URL to the deployed application uses `yourapp` as the hostname (`https://yourapp/`). + +```yaml +stages: + - build + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +# Deploys the container to the GitLab container registry +deploy: + services: + - name: docker:dind + alias: dind + image: docker:20.10.16 + stage: build + script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - docker pull $CI_REGISTRY_IMAGE:latest || true + - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA --tag $CI_REGISTRY_IMAGE:latest . + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + - docker push $CI_REGISTRY_IMAGE:latest + +dast_api: + services: # use services to link your app container to the dast job + - name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + alias: yourapp + +variables: + DAST_API_TARGET_URL: https://yourapp +``` + +Most applications depend on multiple services such as databases or caching services. By default, services defined in the services fields cannot communicate +with each another. To allow communication between services, enable the `FF_NETWORK_PER_BUILD` [feature flag](https://docs.gitlab.com/runner/configuration/feature-flags.html#available-feature-flags). + +```yaml +variables: + FF_NETWORK_PER_BUILD: "true" # enable network per build so all services can communicate on the same network + +services: # use services to link the container to the dast job + - name: mongo:latest + alias: mongo + - name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + alias: yourapp +``` diff --git a/doc/user/application_security/api_security_testing/configuration/variables.md b/doc/user/application_security/api_security_testing/configuration/variables.md new file mode 100644 index 00000000000..ceaaca8c75e --- /dev/null +++ b/doc/user/application_security/api_security_testing/configuration/variables.md @@ -0,0 +1,104 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Available CI/CD variables and configuration files + +## Available CI/CD variables + +| CI/CD variable | Description | +|------------------------------------------------------|--------------------| +| `SECURE_ANALYZERS_PREFIX` | Specify the Docker registry base address from which to download the analyzer. | +| `DAST_API_DISABLED` | Set to 'true' or '1' to disable API security testing scanning. | +| `DAST_API_DISABLED_FOR_DEFAULT_BRANCH` | Set to 'true' or '1' to disable API security testing scanning for only the default (production) branch. | +| `DAST_API_VERSION` | Specify API security testing container version. Defaults to `3`. | +| `DAST_API_IMAGE_SUFFIX` | Specify a container image suffix. Defaults to none. | +| `DAST_API_API_PORT` | Specify the communication port number used by API security testing engine. Defaults to `5500`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367734) in GitLab 15.5. | +| `DAST_API_TARGET_URL` | Base URL of API testing target. | +|[`DAST_API_CONFIG`](#configuration-files) | API security testing configuration file. Defaults to `.gitlab-dast-api.yml`. | +|[`DAST_API_PROFILE`](#configuration-files) | Configuration profile to use during testing. Defaults to `Quick`. | +|[`DAST_API_EXCLUDE_PATHS`](customizing_analyzer_settings.md#exclude-paths) | Exclude API URL paths from testing. | +|[`DAST_API_EXCLUDE_URLS`](customizing_analyzer_settings.md#exclude-urls) | Exclude API URL from testing. | +|[`DAST_API_EXCLUDE_PARAMETER_ENV`](customizing_analyzer_settings.md#exclude-parameters) | JSON string containing excluded parameters. | +|[`DAST_API_EXCLUDE_PARAMETER_FILE`](customizing_analyzer_settings.md#exclude-parameters) | Path to a JSON file containing excluded parameters. | +|[`DAST_API_REQUEST_HEADERS`](customizing_analyzer_settings.md#request-headers) | A comma-separated (`,`) list of headers to include on each scan request. Consider using `DAST_API_REQUEST_HEADERS_BASE64` when storing secret header values in a [masked variable](../../../../ci/variables/index.md#mask-a-cicd-variable), which has character set restrictions. | +|[`DAST_API_REQUEST_HEADERS_BASE64`](customizing_analyzer_settings.md#request-headers) | A comma-separated (`,`) list of headers to include on each scan request, Base64-encoded. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/378440) in GitLab 15.6. | +|[`DAST_API_OPENAPI`](enabling_the_analyzer.md#openapi-specification) | OpenAPI specification file or URL. | +|[`DAST_API_OPENAPI_RELAXED_VALIDATION`](enabling_the_analyzer.md#openapi-specification) | Relax document validation. Default is disabled. | +|[`DAST_API_OPENAPI_ALL_MEDIA_TYPES`](enabling_the_analyzer.md#openapi-specification) | Use all supported media types instead of one when generating requests. Causes test duration to be longer. Default is disabled. | +|[`DAST_API_OPENAPI_MEDIA_TYPES`](enabling_the_analyzer.md#openapi-specification) | Colon (`:`) separated media types accepted for testing. Default is disabled. | +|[`DAST_API_HAR`](enabling_the_analyzer.md#http-archive-har) | HTTP Archive (HAR) file. | +|[`DAST_API_GRAPHQL`](enabling_the_analyzer.md#graphql-schema) | Path to GraphQL endpoint, for example `/api/graphql`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352780) in GitLab 15.4. | +|[`DAST_API_GRAPHQL_SCHEMA`](enabling_the_analyzer.md#graphql-schema) | A URL or filename for a GraphQL schema in JSON format. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352780) in GitLab 15.4. | +|[`DAST_API_POSTMAN_COLLECTION`](enabling_the_analyzer.md#postman-collection) | Postman Collection file. | +|[`DAST_API_POSTMAN_COLLECTION_VARIABLES`](enabling_the_analyzer.md#postman-variables) | Path to a JSON file to extract Postman variable values. The support for comma-separated (`,`) files was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356312) in GitLab 15.1. | +|[`DAST_API_OVERRIDES_FILE`](customizing_analyzer_settings.md#overrides) | Path to a JSON file containing overrides. | +|[`DAST_API_OVERRIDES_ENV`](customizing_analyzer_settings.md#overrides) | JSON string containing headers to override. | +|[`DAST_API_OVERRIDES_CMD`](customizing_analyzer_settings.md#overrides) | Overrides command. | +|[`DAST_API_OVERRIDES_CMD_VERBOSE`](customizing_analyzer_settings.md#overrides) | When set to any value. It shows overrides command output as part of the job output. | +|`DAST_API_PRE_SCRIPT` | Run user command or script before scan session starts. | +|`DAST_API_POST_SCRIPT` | Run user command or script after scan session has finished. | +|[`DAST_API_OVERRIDES_INTERVAL`](customizing_analyzer_settings.md#overrides) | How often to run overrides command in seconds. Defaults to `0` (once). | +|[`DAST_API_HTTP_USERNAME`](customizing_analyzer_settings.md#http-basic-authentication) | Username for HTTP authentication. | +|[`DAST_API_HTTP_PASSWORD`](customizing_analyzer_settings.md#http-basic-authentication) | Password for HTTP authentication. Consider using `DAST_API_HTTP_PASSWORD_BASE64` instead. | +|[`DAST_API_HTTP_PASSWORD_BASE64`](customizing_analyzer_settings.md#http-basic-authentication) | Password for HTTP authentication, base64-encoded. [Introduced](https://gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing-src/-/merge_requests/702) in GitLab 15.4. | +|`DAST_API_SERVICE_START_TIMEOUT` | How long to wait for target API to become available in seconds. Default is 300 seconds. | +|`DAST_API_TIMEOUT` | How long to wait for API responses in seconds. Default is 30 seconds. | + +## Configuration files + +To get you started quickly, GitLab provides the configuration file +[`gitlab-dast-api-config.yml`](https://gitlab.com/gitlab-org/security-products/analyzers/dast/-/blob/master/config/gitlab-dast-api-config.yml). +This file has several testing profiles that perform various numbers of tests. The run time of each +profile increases as the test numbers go up. To use a configuration file, add it to your +repository's root as `.gitlab/gitlab-dast-api-config.yml`. + +### Profiles + +The following profiles are pre-defined in the default configuration file. Profiles +can be added, removed, and modified by creating a custom configuration. + +#### Passive + +- Application Information Check +- Cleartext Authentication Check +- JSON Hijacking Check +- Sensitive Information Check +- Session Cookie Check + +#### Quick + +- Application Information Check +- Cleartext Authentication Check +- FrameworkDebugModeCheck +- HTML Injection Check +- Insecure Http Methods Check +- JSON Hijacking Check +- JSON Injection Check +- Sensitive Information Check +- Session Cookie Check +- SQL Injection Check +- Token Check +- XML Injection Check + +#### Full + +- Application Information Check +- Cleartext AuthenticationCheck +- CORS Check +- DNS Rebinding Check +- Framework Debug Mode Check +- HTML Injection Check +- Insecure Http Methods Check +- JSON Hijacking Check +- JSON Injection Check +- Open Redirect Check +- Sensitive File Check +- Sensitive Information Check +- Session Cookie Check +- SQL Injection Check +- TLS Configuration Check +- Token Check +- XML Injection Check diff --git a/doc/user/application_security/dast_api/img/dast_api_postman_collection_edit_variable.png b/doc/user/application_security/api_security_testing/img/dast_api_postman_collection_edit_variable.png similarity index 100% rename from doc/user/application_security/dast_api/img/dast_api_postman_collection_edit_variable.png rename to doc/user/application_security/api_security_testing/img/dast_api_postman_collection_edit_variable.png diff --git a/doc/user/application_security/dast_api/img/dast_api_postman_environment_edit_variable.png b/doc/user/application_security/api_security_testing/img/dast_api_postman_environment_edit_variable.png similarity index 100% rename from doc/user/application_security/dast_api/img/dast_api_postman_environment_edit_variable.png rename to doc/user/application_security/api_security_testing/img/dast_api_postman_environment_edit_variable.png diff --git a/doc/user/application_security/dast_api/img/dast_api_postman_request_edit.png b/doc/user/application_security/api_security_testing/img/dast_api_postman_request_edit.png similarity index 100% rename from doc/user/application_security/dast_api/img/dast_api_postman_request_edit.png rename to doc/user/application_security/api_security_testing/img/dast_api_postman_request_edit.png diff --git a/doc/user/application_security/api_security_testing/index.md b/doc/user/application_security/api_security_testing/index.md new file mode 100644 index 00000000000..e51b6ddfc90 --- /dev/null +++ b/doc/user/application_security/api_security_testing/index.md @@ -0,0 +1,85 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# API security testing analyzer + +DETAILS: +**Tier:** Ultimate +**Offering:** GitLab.com, Self-managed, GitLab Dedicated + +> - API security testing analyzer [became the default analyzer for on-demand API security testing scans](https://gitlab.com/groups/gitlab-org/-/epics/4254) in GitLab 15.6. +> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/457449) from **DAST API analyzer** to **API security testing analyzer** in GitLab 17.0. + +Perform Dynamic Application Security Testing (DAST) of web APIs to help discover bugs and potential +security issues that other QA processes may miss. Use API security testing in addition to +other [GitLab Secure](../index.md) security scanners and your own test processes. You can run DAST +API tests either as part your CI/CD workflow, [on-demand](../dast/on-demand_scan.md), or both. + +WARNING: +Do not run API security testing against a production server. Not only can it perform _any_ function that +the API can, it may also trigger bugs in the API. This includes actions like modifying and deleting +data. Only run API security testing against a test server. + +API security testing can test the following web API types: + +- REST API +- SOAP +- GraphQL +- Form bodies, JSON, or XML + +## When API security testing scans run + +When run in your CI/CD pipeline, API security testing scanning runs in the `dast` stage by default. To ensure +API security testing scanning examines the latest code, ensure your CI/CD pipeline deploys changes to a test +environment in a stage before the `dast` stage. + +If your pipeline is configured to deploy to the same web server on each run, running a pipeline +while another is still running could cause a race condition in which one pipeline overwrites the +code from another. The API to be scanned should be excluded from changes for the duration of a +API security testing scan. The only changes to the API should be from the API security testing scanner. Changes made to the +API (for example, by users, scheduled tasks, database changes, code changes, other pipelines, or +other scanners) during a scan could cause inaccurate results. + +## Example API security testing scanning configurations + +The following projects demonstrate API security testing scanning: + +- [Example OpenAPI v2 Specification project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/openapi-example) +- [Example HTTP Archive (HAR) project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/har-example) +- [Example Postman Collection project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/postman-example) +- [Example GraphQL project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/graphql-example) +- [Example SOAP project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/soap-example) +- [Authentication Token using Selenium](https://gitlab.com/gitlab-org/security-products/demos/api-dast/auth-token-selenium) + +## Get support or request an improvement + +To get support for your particular problem, use the [getting help channels](https://about.gitlab.com/get-help/). + +The [GitLab issue tracker on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues) is the right place for bugs and feature proposals about API Security and API security testing. +Use `~"Category:API Security"` [label](../../../development/labels/index.md) when opening a new issue regarding API security testing to ensure it is quickly reviewed by the right people. Refer to our [review response SLO](https://handbook.gitlab.com/handbook/engineering/workflow/code-review/#review-response-slo) to understand when you should receive a response. + +[Search the issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues) for similar entries before submitting your own, there's a good chance somebody else had the same issue or feature proposal. Show your support with an emoji reaction or join the discussion. + +When experiencing a behavior not working as expected, consider providing contextual information: + +- GitLab version if using a self-managed instance. +- `.gitlab-ci.yml` job definition. +- Full job console output. +- Scanner log file available as a job artifact named `gl-api-security-scanner.log`. + +WARNING: +**Sanitize data attached to a support issue**. Remove sensitive information, including: credentials, passwords, tokens, keys, and secrets. + +## Glossary + +- Assert: Assertions are detection modules used by checks to trigger a vulnerability. Many assertions have + configurations. A check can use multiple Assertions. For example, Log Analysis, Response Analysis, + and Status Code are common Assertions used together by checks. Checks with multiple Assertions + allow them to be turned on and off. +- Check: Performs a specific type of test, or performed a check for a type of vulnerability. For + example, the SQL Injection Check performs DAST testing for SQL Injection vulnerabilities. The API security testing scanner is comprised of several checks. Checks can be turned on and off in a profile. +- Profile: A configuration file has one or more testing profiles, or sub-configurations. You may + have a profile for feature branches and another with extra testing for a main branch. diff --git a/doc/user/application_security/api_security_testing/performance.md b/doc/user/application_security/api_security_testing/performance.md new file mode 100644 index 00000000000..8c3792dd3ec --- /dev/null +++ b/doc/user/application_security/api_security_testing/performance.md @@ -0,0 +1,219 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Performance tuning and testing speed + +Security tools that perform dynamic analysis testing, such as API security testing, perform testing by sending requests to an instance of your running application. The requests are engineered to test for specific vulnerabilities that might exist in your application. The speed of a dynamic analysis test depends on the following: + +- How many requests per second can be sent to your application by our tooling +- How fast your application responds to requests +- How many requests must be sent to test the application + - How many operations your API is comprised of + - How many fields are in each operation (think JSON bodies, headers, query string, cookies, etc.) + +If the API security testing job still takes longer than expected reach after following the advice in this performance guide, reach out to support for further assistance. + +## Diagnosing performance issues + +The first step to resolving performance issues is to understand what is contributing to the slower-than-expected testing time. Some common issues we see are: + +- API security testing is running on a low-vCPU runner +- The application deployed to a slow/single-CPU instance and is not able to keep up with the testing load +- The application contains a slow operation that impacts the overall test speed (> 1/2 second) +- The application contains an operation that returns a large amount of data (> 500K+) +- The application contains a large number of operations (> 40) + +### The application contains a slow operation that impacts the overall test speed (> 1/2 second) + +The API security testing job output contains helpful information about how fast we are testing, how fast each operation being tested responds, and summary information. Let's take a look at some sample output to see how it can be used in tracking down performance issues: + +```shell +DAST API: Loaded 10 operations from: assets/har-large-response/large_responses.har +DAST API: +DAST API: Testing operation [1/10]: 'GET http://target:7777/api/large_response_json'. +DAST API: - Parameters: (Headers: 4, Query: 0, Body: 0) +DAST API: - Request body size: 0 Bytes (0 bytes) +DAST API: +DAST API: Finished testing operation 'GET http://target:7777/api/large_response_json'. +DAST API: - Excluded Parameters: (Headers: 0, Query: 0, Body: 0) +DAST API: - Performed 767 requests +DAST API: - Average response body size: 130 MB +DAST API: - Average call time: 2 seconds and 82.69 milliseconds (2.082693 seconds) +DAST API: - Time to complete: 14 minutes, 8 seconds and 788.36 milliseconds (848.788358 seconds) +``` + +This job console output snippet starts by telling us how many operations were found (10), followed by notifications that testing has started on a specific operation and a summary of the operation has been completed. The summary is the most interesting part of this log output. In the summary, we can see that it took API security testing 767 requests to fully test this operation and its related fields. We can also see that the average response time was 2 seconds and the time to complete was 14 minutes for this one operation. + +An average response time of 2 seconds is a good initial indicator that this specific operation takes a long time to test. Further, we can see that the response body size is quite large. The large body size is the culprit here, transferring that much data on each request is what takes the majority of that 2 seconds. + +For this issue, the team might decide to: + +- Use a runner with more vCPUs, as this allows API security testing to parallelize the work being performed. This helps lower the test time, but getting the test down under 10 minutes might still be problematic without moving to a high CPU machine due to how long the operation takes to test. While larger runners are more costly, you also pay for less minutes if the job executions are quicker. +- [Exclude this operation](#excluding-slow-operations) from API security testing. While this is the simplest, it has the downside of a gap in security test coverage. +- [Exclude the operation from feature branch API security testing, but include it in the default branch test](#excluding-operations-in-feature-branches-but-not-default-branch). +- [Split up API security testing into multiple jobs](#splitting-a-test-into-multiple-jobs). + +The likely solution is to use a combination of these solutions to reach an acceptable test time, assuming your team's requirements are in the 5-7 minute range. + +## Addressing performance issues + +The following sections document various options for addressing performance issues for API security testing: + +- [Using a larger runner](#using-a-larger-runner) +- [Excluding slow operations](#excluding-slow-operations) +- [Splitting a test into multiple jobs](#splitting-a-test-into-multiple-jobs) +- [Excluding operations in feature branches, but not default branch](#excluding-operations-in-feature-branches-but-not-default-branch) + +### Using a larger runner + +One of the easiest performance boosts can be achieved using a [larger runner](../../../ci/runners/hosted_runners/linux.md#machine-types-available-for-linux-x86-64) with API security testing. This table shows statistics collected during benchmarking of a Java Spring Boot REST API. In this benchmark, the target and API security testing share a single runner instance. + +| Hosted runner on Linux tag | Requests per Second | +|------------------------------------|-----------| +| `saas-linux-small-amd64` (default) | 255 | +| `saas-linux-medium-amd64` | 400 | + +As we can see from this table, increasing the size of the runner and vCPU count can have a large impact on testing speed/performance. + +Here is an example job definition for API security testing that adds a `tags` section to use the medium SaaS runner on Linux. The job extends the job definition included through the API security testing template. + +```yaml +dast_api: + tags: + - saas-linux-medium-amd64 +``` + +In the `gl-api-security-scanner.log` file you can search for the string `Starting work item processor` to inspect the reported max DOP (degree of parallelism). The max DOP should be greater than or equal to the number of vCPUs assigned to the runner. If unable to identify the problem, open a ticket with support to assist. + +Example log entry: + +`17:00:01.084 [INF] Starting work item processor with 4 max DOP` + +### Excluding slow operations + +In the case of one or two slow operations, the team might decide to skip testing the operations. Excluding the operation is done using the `DAST_API_EXCLUDE_PATHS` configuration [variable as explained in this section.](configuration/customizing_analyzer_settings.md#exclude-paths) + +In this example, we have an operation that returns a large amount of data. The operation is `GET http://target:7777/api/large_response_json`. To exclude it we provide the `DAST_API_EXCLUDE_PATHS` configuration variable with the path portion of our operation URL `/api/large_response_json`. + +To verify the operation is excluded, run the API security testing job and review the job console output. It includes a list of included and excluded operations at the end of the test. + +```yaml +dast_api: + variables: + DAST_API_EXCLUDE_PATHS: /api/large_response_json +``` + +Excluding operations from testing could allow some vulnerabilities to go undetected. +{: .alert .alert-warning} + +### Splitting a test into multiple jobs + +Splitting a test into multiple jobs is supported by API security testing through the use of [`DAST_API_EXCLUDE_PATHS`](configuration/customizing_analyzer_settings.md#exclude-paths) and [`DAST_API_EXCLUDE_URLS`](configuration/customizing_analyzer_settings.md#exclude-urls). When splitting a test up, a good pattern is to disable the `dast_api` job and replace it with two jobs with identifying names. In this example we have two jobs, each job is testing a version of the API, so our names reflect that. However, this technique can be applied to any situation, not just with versions of an API. + +The rules we are using in the `dast_api_v1` and `dast_api_v2` jobs are copied from the [API security testing template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml). + +```yaml +# Disable the main dast_api job +dast_api: + rules: + - if: $CI_COMMIT_BRANCH + when: never + +dast_api_v1: + extends: dast_api + variables: + DAST_API_EXCLUDE_PATHS: /api/v1/** + rules: + - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1' + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DAST_API_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH + +dast_api_v2: + variables: + DAST_API_EXCLUDE_PATHS: /api/v2/** + rules: + - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1' + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DAST_API_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH +``` + +### Excluding operations in feature branches, but not default branch + +In the case of one or two slow operations, the team might decide to skip testing the operations, or exclude them from feature branch tests, but include them for default branch tests. Excluding the operation is done using the `DAST_API_EXCLUDE_PATHS` configuration [variable as explained in this section.](configuration/customizing_analyzer_settings.md#exclude-paths) + +In this example, we have an operation that returns a large amount of data. The operation is `GET http://target:7777/api/large_response_json`. To exclude it we provide the `DAST_API_EXCLUDE_PATHS` configuration variable with the path portion of our operation URL `/api/large_response_json`. Our configuration disables the main `dast_api` job and creates two new jobs `dast_api_main` and `dast_api_branch`. The `dast_api_branch` is set up to exclude the long operation and only run on non-default branches (for example, feature branches). The `dast_api_main` branch is set up to only execute on the default branch (`main` in this example). The `dast_api_branch` jobs run faster, allowing for quick development cycles, while the `dast_api_main` job which only runs on default branch builds, takes longer to run. + +To verify the operation is excluded, run the API security testing job and review the job console output. It includes a list of included and excluded operations at the end of the test. + +```yaml +# Disable the main job so we can create two jobs with +# different names +dast_api: + rules: + - if: $CI_COMMIT_BRANCH + when: never + +# API security testing for feature branch work, excludes /api/large_response_json +dast_api_branch: + extends: dast_api + variables: + DAST_API_EXCLUDE_PATHS: /api/large_response_json + rules: + - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1' + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DAST_API_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: never + - if: $CI_COMMIT_BRANCH + +# API security testing for default branch (main in our case) +# Includes the long running operations +dast_api_main: + extends: dast_api + rules: + - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1' + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DAST_API_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH +``` diff --git a/doc/user/application_security/api_security_testing/troubleshooting.md b/doc/user/application_security/api_security_testing/troubleshooting.md new file mode 100644 index 00000000000..78bfb1feeed --- /dev/null +++ b/doc/user/application_security/api_security_testing/troubleshooting.md @@ -0,0 +1,302 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Troubleshooting + +## API security testing job times out after N hours + +For larger repositories, the API security testing job could time out on the [small hosted runner on Linux](../../../ci/runners/hosted_runners/linux.md#machine-types-available-for-linux-x86-64), which is set per default. If this happens in your jobs, you should scale up to a [larger runner](performance.md#using-a-larger-runner). + +See the following documentation sections for assistance: + +- [Performance tuning and testing speed](performance.md#performance-tuning-and-testing-speed) +- [Using a larger Runner](performance.md#using-a-larger-runner) +- [Excluding operations by path](configuration/customizing_analyzer_settings.md#exclude-paths) +- [Excluding slow operations](performance.md#excluding-slow-operations) + +## API security testing job takes too long to complete + +See [Performance Tuning and Testing Speed](performance.md#performance-tuning-and-testing-speed) + +## Error: `Error waiting for DAST API 'http://127.0.0.1:5000' to become available` + +A bug exists in versions of the API security testing analyzer prior to v1.6.196 that can cause a background process to fail under certain conditions. The solution is to update to a newer version of the API security testing analyzer. + +The version information can be found in the job details for the `dast_api` job. + +If the issue is occurring with versions v1.6.196 or greater, contact Support and provide the following information: + +1. Reference this troubleshooting section and ask for the issue to be escalated to the Dynamic Analysis Team. +1. The full console output of the job. +1. The `gl-api-security-scanner.log` file available as a job artifact. In the right-hand panel of the job details page, select the **Browse** button. +1. The `dast_api` job definition from your `.gitlab-ci.yml` file. + +**Error message** + +- In [GitLab 15.6 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/376078), `Error waiting for DAST API 'http://127.0.0.1:5000' to become available` +- In GitLab 15.5 and earlier, `Error waiting for API Security 'http://127.0.0.1:5000' to become available`. + +## `Failed to start scanner session (version header not found)` + +The API security testing engine outputs an error message when it cannot establish a connection with the scanner application component. The error message is shown in the job output window of the `dast_api` job. A common cause of this issue is changing the `DAST_API_API` variable from its default. + +**Error message** + +- `Failed to start scanner session (version header not found).` + +**Solution** + +- Remove the `DAST_API_API` variable from the `.gitlab-ci.yml` file. The value inherits from the API security testing CI/CD template. We recommend this method instead of manually setting a value. +- If removing the variable is not possible, check to see if this value has changed in the latest version of the [API security testing CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml). If so, update the value in the `.gitlab-ci.yml` file. + +## `Failed to start session with scanner. Please retry, and if the problem persists reach out to support.` + +The API security testing engine outputs an error message when it cannot establish a connection with the scanner application component. The error message is shown in the job output window of the `dast_api` job. A common cause for this issue is that the background component cannot use the selected port as it's already in use. This error can occur intermittently if timing plays a part (race condition). This issue occurs most often with Kubernetes environments when other services are mapped into the container causing port conflicts. + +Before proceeding with a solution, it is important to confirm that the error message was produced because the port was already taken. To confirm this was the cause: + +1. Go to the job console. + +1. Look for the artifact `gl-api-security-scanner.log`. You can either download all artifacts by selecting **Download** and then search for the file, or directly start searching by selecting **Browse**. + +1. Open the file `gl-api-security-scanner.log` in a text editor. + +1. If the error message was produced because the port was already taken, you should see in the file a message like the following: + +- In [GitLab 15.5 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/367734): + + ```log + Failed to bind to address http://127.0.0.1:5500: address already in use. + ``` + +- In GitLab 15.4 and earlier: + + ```log + Failed to bind to address http://[::]:5000: address already in use. + ``` + +The text `http://[::]:5000` in the previous message could be different in your case, for instance it could be `http://[::]:5500` or `http://127.0.0.1:5500`. As long as the remaining parts of the error message are the same, it is safe to assume the port was already taken. + +If you did not find evidence that the port was already taken, check other troubleshooting sections which also address the same error message shown in the job console output. If there are no more options, feel free to [get support or request an improvement](index.md#get-support-or-request-an-improvement) through the proper channels. + +Once you have confirmed the issue was produced because the port was already taken. Then, [GitLab 15.5 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/367734) introduced the configuration variable `DAST_API_API_PORT`. This configuration variable allows setting a fixed port number for the scanner background component. + +**Solution** + +1. Ensure your `.gitlab-ci.yml` file defines the configuration variable `DAST_API_API_PORT`. +1. Update the value of `DAST_API_API_PORT` to any available port number greater than 1024. We recommend checking that the new value is not in used by GitLab. See the full list of ports used by GitLab in [Package defaults](../../../administration/package_information/defaults.md#ports) + +## `Application cannot determine the base URL for the target API` + +The API security testing engine outputs an error message when it cannot determine the target API after inspecting the OpenAPI document. This error message is shown when the target API has not been set in the `.gitlab-ci.yml` file, it is not available in the `environment_url.txt` file, and it could not be computed using the OpenAPI document. + +There is a order of precedence in which the API security testing engine tries to get the target API when checking the different sources. First, it tries to use the `DAST_API_TARGET_URL`. If the environment variable has not been set, then the API security testing engine attempts to use the `environment_url.txt` file. If there is no file `environment_url.txt`, then the API security testing engine uses the OpenAPI document contents and the URL provided in `DAST_API_OPENAPI` (if a URL is provided) to try to compute the target API. + +The best-suited solution depends on whether or not your target API changes for each deployment. In static environments, the target API is the same for each deployment, in this case refer to the [static environment solution](#static-environment-solution). If the target API changes for each deployment a [dynamic environment solution](#dynamic-environment-solutions) should be applied. + +## API security testing job excludes some paths from operations + +If you find that some paths are being excluded from operations, ensure that the `consumes` array is defined and has a valid type in the target definition JSON file. This is required. + +See the [example project target definition file](https://gitlab.com/gitlab-org/security-products/demos/api-dast/openapi-example/-/blob/12e2b039d08208f1dd38a1e7c52b0bda848bb449/rest_target_openapi.json?plain=1#L13) where the `consumes` array is defined. + +### Static environment solution + +This solution is for pipelines in which the target API URL doesn't change (is static). + +**Add environmental variable** + +For environments where the target API remains the same, we recommend you specify the target URL by using the `DAST_API_TARGET_URL` environment variable. In your `.gitlab-ci.yml`, add a variable `DAST_API_TARGET_URL`. The variable must be set to the base URL of API testing target. For example: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OPENAPI: test-api-specification.json +``` + +### Dynamic environment solutions + +In a dynamic environment your target API changes for each different deployment. In this case, there is more than one possible solution, we recommend you use the `environment_url.txt` file when dealing with dynamic environments. + +**Use environment_url.txt** + +To support dynamic environments in which the target API URL changes during each pipeline, API security testing engine supports the use of an `environment_url.txt` file that contains the URL to use. This file is not checked into the repository, instead it's created during the pipeline by the job that deploys the test target and collected as an artifact that can be used by later jobs in the pipeline. The job that creates the `environment_url.txt` file must run before the API security testing engine job. + +1. Modify the test target deployment job adding the base URL in an `environment_url.txt` file at the root of your project. +1. Modify the test target deployment job collecting the `environment_url.txt` as an artifact. + +Example: + +```yaml +deploy-test-target: + script: + # Perform deployment steps + # Create environment_url.txt (example) + - echo http://${CI_PROJECT_ID}-${CI_ENVIRONMENT_SLUG}.example.org > environment_url.txt + + artifacts: + paths: + - environment_url.txt +``` + +## Use OpenAPI with an invalid schema + +There are cases where the document is autogenerated with an invalid schema or cannot be edited manually in a timely manner. In those scenarios, the API security testing is able to perform a relaxed validation by setting the variable `DAST_API_OPENAPI_RELAXED_VALIDATION`. We recommend providing a fully compliant OpenAPI document to prevent unexpected behaviors. + +### Edit a non-compliant OpenAPI file + +To detect and correct elements that don't comply with the OpenAPI specifications, we recommend using an editor. An editor commonly provides document validation, and suggestions to create a schema-compliant OpenAPI document. Suggested editors include: + +| Editor | OpenAPI 2.0 | OpenAPI 3.0.x | OpenAPI 3.1.x | +| -- | -- | -- | -- | +| [Swagger Editor](https://editor.swagger.io/) | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | **{dotted-circle}** YAML, JSON | +| [Stoplight Studio](https://stoplight.io/studio) | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | + +If your OpenAPI document is generated manually, load your document in the editor and fix anything that is non-compliant. If your document is generated automatically, load it in your editor to identify the issues in the schema, then go to the application and perform the corrections based on the framework you are using. + +### Enable OpenAPI relaxed validation + +Relaxed validation is meant for cases when the OpenAPI document cannot meet OpenAPI specifications, but it still has enough content to be consumed by different tools. A validation is performed but less strictly in regards to document schema. + +API security testing can still try to consume an OpenAPI document that does not fully comply with OpenAPI specifications. To instruct API security testing to perform a relaxed validation, set the variable `DAST_API_OPENAPI_RELAXED_VALIDATION` to any value, for example: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_PROFILE: Quick + DAST_API_TARGET_URL: http://test-deployment/ + DAST_API_OPENAPI: test-api-specification.json + DAST_API_OPENAPI_RELAXED_VALIDATION: 'On' +``` + +## `No operation in the OpenAPI document is consuming any supported media type` + +API security testing uses the specified media types in the OpenAPI document to generate requests. If no request can be created due to the lack of supported media types, then an error is thrown. + +**Error message** + +- `Error, no operation in the OpenApi document is consuming any supported media type. Check 'OpenAPI Specification' to check the supported media types.` + +**Solution** + +1. Review supported media types in the [OpenAPI Specification](configuration/enabling_the_analyzer.md#openapi-specification) section. +1. Edit your OpenAPI document, allowing at least a given operation to accept any of the supported media types. Alternatively, a supported media type could be set in the OpenAPI document level and get applied to all operations. This step may require changes in your application to ensure the supported media type is accepted by the application. + +## ``Error, error occurred trying to download ``: There was an error when retrieving content from Uri:' '. Error:The SSL connection could not be established, see inner exception.`` + +API security testing is compatible with a broad range of TLS configurations, including outdated protocols and ciphers. +Despite broad support, you might encounter connection errors. This error occurs because API security testing could not establish a secure connection with the server at the given URL. + +To resolve the issue: + +If the host in the error message supports non-TLS connections, change `https://` to `http://` in your configuration. +For example, if an error occurs with the following configuration: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_TARGET_URL: https://test-deployment/ + DAST_API_OPENAPI: https://specs/openapi.json +``` + +Change the prefix of `DAST_API_OPENAPI` from `https://` to `http://`: + +```yaml +stages: + - dast + +include: + - template: DAST-API.gitlab-ci.yml + +variables: + DAST_API_TARGET_URL: https://test-deployment/ + DAST_API_OPENAPI: http://specs/openapi.json +``` + +If you cannot use a non-TLS connection to access the URL, contact the Support team for help. + +You can expedite the investigation with the [testssl.sh tool](https://testssl.sh/). From a machine with a bash shell and connectivity to the affected server: + +1. Download the latest release `zip` or `tar.gz` file and extract from . +1. Run `./testssl.sh --log https://specs`. +1. Attach the log file to your support ticket. + +## `ERROR: Job failed: failed to pull image` + +This error message occurs when pulling an image from a container registry that requires authentication to access (it is not public). + +In the job console output the error looks like: + +```log +Running with gitlab-runner 15.6.0~beta.186.ga889181a (a889181a) + on blue-2.shared.runners-manager.gitlab.com/default XxUrkriX +Resolving secrets +00:00 +Preparing the "docker+machine" executor +00:06 +Using Docker executor with image registry.gitlab.com/security-products/api-security:2 ... +Starting service registry.example.com/my-target-app:latest ... +Pulling docker image registry.example.com/my-target-app:latest ... +WARNING: Failed to pull image with policy "always": Error response from daemon: Get https://registry.example.com/my-target-app/manifests/latest: unauthorized (manager.go:237:0s) +ERROR: Job failed: failed to pull image "registry.example.com/my-target-app:latest" with specified policies [always]: Error response from daemon: Get https://registry.example.com/my-target-app/manifests/latest: unauthorized (manager.go:237:0s) +``` + +**Error message** + +- In GitLab 15.9 and earlier, `ERROR: Job failed: failed to pull image` followed by `Error response from daemon: Get IMAGE: unauthorized`. + +**Solution** + +Authentication credentials are provided using the methods outlined in the [Access an image from a private container registry](../../../ci/docker/using_docker_images.md#access-an-image-from-a-private-container-registry) documentation section. The method used is dictated by your container registry provider and its configuration. If your using a container registry provided by a 3rd party, such as a cloud provider (Azure, Google Could (GCP), AWS and so on), check the providers documentation for information on how to authenticate to their container registries. + +The following example uses the [statically defined credentials](../../../ci/docker/using_docker_images.md#use-statically-defined-credentials) authentication method. In this example the container registry is `registry.example.com` and image is `my-target-app:latest`. + +1. Read how to [Determine your `DOCKER_AUTH_CONFIG` data](../../../ci/docker/using_docker_images.md#determine-your-docker_auth_config-data) to understand how to compute the variable value for `DOCKER_AUTH_CONFIG`. The configuration variable `DOCKER_AUTH_CONFIG` contains the Docker JSON configuration to provide the appropriate authentication information. For example, to access private container registry: `registry.example.com` with the credentials `abcdefghijklmn`, the Docker JSON looks like: + + ```json + { + "auths": { + "registry.example.com": { + "auth": "abcdefghijklmn" + } + } + } + ``` + +1. Add the `DOCKER_AUTH_CONFIG` as a CI/CD variable. Instead of adding the configuration variable directly in your `.gitlab-ci.yml`file you should create a project [CI/CD variable](../../../ci/variables/index.md#for-a-project). +1. Rerun your job, and the statically-defined credentials are now used to sign in to the private container registry `registry.example.com`, and let you pull the image `my-target-app:latest`. If succeeded the job console shows an output like: + + ```log + Running with gitlab-runner 15.6.0~beta.186.ga889181a (a889181a) + on blue-4.shared.runners-manager.gitlab.com/default J2nyww-s + Resolving secrets + 00:00 + Preparing the "docker+machine" executor + 00:56 + Using Docker executor with image registry.gitlab.com/security-products/api-security:2 ... + Starting service registry.example.com/my-target-app:latest ... + Authenticating with credentials from $DOCKER_AUTH_CONFIG + Pulling docker image registry.example.com/my-target-app:latest ... + Using docker image sha256:139c39668e5e4417f7d0eb0eeb74145ba862f4f3c24f7c6594ecb2f82dc4ad06 for registry.example.com/my-target-app:latest with digest registry.example.com/my-target- + app@sha256:2b69fc7c3627dbd0ebaa17674c264fcd2f2ba21ed9552a472acf8b065d39039c ... + Waiting for services to be up and running (timeout 30 seconds)... + ``` diff --git a/doc/user/application_security/dast_api/checks/application_information_check.md b/doc/user/application_security/dast_api/checks/application_information_check.md index 27d95437e43..75b73caa2c7 100644 --- a/doc/user/application_security/dast_api/checks/application_information_check.md +++ b/doc/user/application_security/dast_api/checks/application_information_check.md @@ -1,24 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/application_information_check.md' +remove_date: '2024-07-30' --- -# Application information disclosure +This document was moved to [another location](../../api_security_testing/checks/application_information_check.md). -## Description - -Application information disclosure check. This includes information such as version numbers, database error messages, stack traces. - -## Remediation - -Application information disclosure is an application weakness where an application reveals sensitive data, such as technical details of the web application or environment. Application data may be used by an attacker to exploit the target web application, its hosting network, or its users. Therefore, leakage of sensitive data should be limited or prevented whenever possible. Information disclosure, in its most common form, is the result of one or more of the following conditions: a failure to scrub out HTML or script comments containing sensitive information or improper application or server configurations. - -Failure to scrub HTML or script comments prior to a push to the production environment can result in the leak of sensitive, contextual, information such as server directory structure, SQL query structure, and internal network information. Often a developer will leave comments within the HTML and script code to help facilitate the debugging or integration process during the pre-production phase. Although there is no harm in allowing developers to include inline comments within the content they develop, these comments should all be removed prior to the content's public release. - -Software version numbers and verbose error messages (such as ASP.NET version numbers) are examples of improper server configurations. This information is useful to an attacker by providing detailed insight as to the framework, languages, or pre-built functions being utilized by a web application. Most default server configurations provide software version numbers and verbose error messages for debugging and troubleshooting purposes. Configuration changes can be made to disable these features, preventing the display of this information. - -## Links - -- [OWASP](https://owasp.org/Top10/A05_2021-Security_Misconfiguration) -- [CWE](https://cwe.mitre.org/data/definitions/200.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/authentication_token_check.md b/doc/user/application_security/dast_api/checks/authentication_token_check.md index 1bd5e8fdb61..707e3e11359 100644 --- a/doc/user/application_security/dast_api/checks/authentication_token_check.md +++ b/doc/user/application_security/dast_api/checks/authentication_token_check.md @@ -1,29 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/authentication_token_check.md' +remove_date: '2024-07-30' --- -# Authentication token +This document was moved to [another location](../../api_security_testing/checks/authentication_token_check.md). -## Description - -Perform various authentication token checks such as removing the token or changing to an invalid value. - -## Remediation - -API tokens must be unpredictable (random enough) to prevent guessing attacks, where an attacker is able to guess or predict a valid API Token through statistical analysis techniques. For this purpose, a good PRNG (Pseudo Random Number Generator) must be used. - -The authentication token may have been: - -- modified to an invalid value. -- removed from request. -- not match length requirements. -- configured as a signature. - -An API operation failed to property restrict access using an authentication token. This allows an attacker to bypass authentication gaining access to information or even the ability to modify data. - -## Links - -- [OWASP](https://owasp.org/Top10/A07_2021-Identification_and_Authentication_Failures) -- [CWE](https://cwe.mitre.org/data/definitions/285.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/cleartext_authentication_check.md b/doc/user/application_security/dast_api/checks/cleartext_authentication_check.md index e8a6ae4e4e5..e21e32fa97e 100644 --- a/doc/user/application_security/dast_api/checks/cleartext_authentication_check.md +++ b/doc/user/application_security/dast_api/checks/cleartext_authentication_check.md @@ -1,20 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/cleartext_authentication_check.md' +remove_date: '2024-07-30' --- -# Cleartext authentication +This document was moved to [another location](../../api_security_testing/checks/cleartext_authentication_check.md). -## Description - -This check looks for cleartext authentication such as HTTP Basic auth with no-TLS. - -## Remediation - -Authentication credentials are transported via unencrypted channel (HTTP). This exposes the transmitted credentials to any attacker who can monitor (sniff) the network traffic during transmission. Sensitive information such as credentials should always be transmitted via encrypted channels such as HTTPS. - -## Links - -- [OWASP](https://owasp.org/Top10/A02_2021-Cryptographic_Failures) -- [CWE](https://cwe.mitre.org/data/definitions/319.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/cors_check.md b/doc/user/application_security/dast_api/checks/cors_check.md index eb22d0a8211..b568c872974 100644 --- a/doc/user/application_security/dast_api/checks/cors_check.md +++ b/doc/user/application_security/dast_api/checks/cors_check.md @@ -1,20 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/cors_check.md' +remove_date: '2024-07-30' --- -# CORS +This document was moved to [another location](../../api_security_testing/checks/cors_check.md). -## Description - -Check for CORS misconfiguration including overly permissive white-lists of accepted Origin headers or failure to validate Origin header. Also checks for allowing credentials on potentially invalid or dangerous Origins and missing headers that could potentially result in cache poisoning. - -## Remediation - -A misconfigured CORS implementation may be overly permissive in which domains should be trusted and at what level of trust. This could allow an untrusted domain to forge the Origin header and launch various types of attacks such as cross-site request forgery or cross-site scripting. An attacker could potentially steal a victim's credentials or send malicious requests on behalf of a victim. The victim may not even be aware that an attack is being launched. - -## Links - -- [OWASP](https://owasp.org/Top10/A01_2021-Broken_Access_Control) -- [CWE](https://cwe.mitre.org/data/definitions/942.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/dns_rebinding_check.md b/doc/user/application_security/dast_api/checks/dns_rebinding_check.md index ca80746928c..c3b9cbb89c7 100644 --- a/doc/user/application_security/dast_api/checks/dns_rebinding_check.md +++ b/doc/user/application_security/dast_api/checks/dns_rebinding_check.md @@ -1,20 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/dns_rebinding_check.md' +remove_date: '2024-07-30' --- -# DNS rebinding +This document was moved to [another location](../../api_security_testing/checks/dns_rebinding_check.md). -## Description - -Check for DNS rebinding. This check verifies that the host checks that the HOST header of the request exists and matches the expected name of the host to avoid attacks via malicious DNS entries. - -## Remediation - -DNS rebinding allows a malicious host to spoof or redirect a request to an alternate IP address, potentially allowing an attacker to bypass security authentication or authorization. DNS resolution on its own does not properly constitute a valid authentication mechanism. Servers should validate that the Host header of the request matches the expected hostname of the server. In cases where the hostname is missing or does not match the expected value, the server should return a 400. The X-Forwarded-Host header is sometimes used instead of the Host header in cases where the request is being forwarded. In these cases, the X-Forwarded-Host header should also be validated if it is being used to determine the Host of the original request. - -## Links - -- [OWASP](https://owasp.org/Top10/A05_2021-Security_Misconfiguration) -- [CWE](https://cwe.mitre.org/data/definitions/350.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/framework_debug_mode_check.md b/doc/user/application_security/dast_api/checks/framework_debug_mode_check.md index 4e8ef50d859..5f4c13bed54 100644 --- a/doc/user/application_security/dast_api/checks/framework_debug_mode_check.md +++ b/doc/user/application_security/dast_api/checks/framework_debug_mode_check.md @@ -1,21 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/framework_debug_mode_check.md' +remove_date: '2024-07-30' --- -# Framework debug mode +This document was moved to [another location](../../api_security_testing/checks/framework_debug_mode_check.md). -## Description - -Checks to see if debug mode is enabled in various frameworks such as Flask and ASP.NET. This check has a low false positive rate. - -## Remediation - -The Flask or ASP .NET framework was identified with debug mode enabled. This allows an attacker the ability to download any file on the file system and other capabilities. This is a high severity issue that is easy for an attacker to exploit. - -## Links - -- [OWASP](https://owasp.org/Top10/A05_2021-Security_Misconfiguration) -- [CWE-23: Relative Path Traversal](https://cwe.mitre.org/data/definitions/23.html) -- [CWE-285: Improper Authorization](https://cwe.mitre.org/data/definitions/285.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/heartbleed_open_ssl_check.md b/doc/user/application_security/dast_api/checks/heartbleed_open_ssl_check.md index b6abc3691b4..3a95dfbc9a3 100644 --- a/doc/user/application_security/dast_api/checks/heartbleed_open_ssl_check.md +++ b/doc/user/application_security/dast_api/checks/heartbleed_open_ssl_check.md @@ -1,22 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/heartbleed_open_ssl_check.md' +remove_date: '2024-07-30' --- -# Heartbleed OpenSSL vulnerability +This document was moved to [another location](../../api_security_testing/checks/heartbleed_open_ssl_check.md). -## Description - -Check for Heartbleed OpenSSL vulnerability. - -## Remediation - -The Heartbleed vulnerability is a serious bug in the popular OpenSSL cryptographic library. OpenSSL is used to encrypt and decrypt communications and secure the Internet traffic. This vulnerability allows the attacker to steal protected information, which should not be accessible under other circumstance such as secret keys that are used to encrypt sensitive information. - -Anyone on with access to the target API can use the Heartbleed vulnerability to read the memory from protected systems taking advantage of vulnerable versions of OpenSSL library. - -## Links - -- [OWASP](https://owasp.org/Top10/A06_2021-Vulnerable_and_Outdated_Components) -- [CWE](https://cwe.mitre.org/data/definitions/119.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/html_injection_check.md b/doc/user/application_security/dast_api/checks/html_injection_check.md index ab273bbae2b..60cc49fd7c3 100644 --- a/doc/user/application_security/dast_api/checks/html_injection_check.md +++ b/doc/user/application_security/dast_api/checks/html_injection_check.md @@ -1,28 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/html_injection_check.md' +remove_date: '2024-07-30' --- -# HTML injection +This document was moved to [another location](../../api_security_testing/checks/html_injection_check.md). -## Description - -Check for XSS via HTML injection into all fields that support strings. This includes portions of the HTTP request such as path, query, headers and also body parameters such as XML fields, JSON fields, etc. Detection is performed by monitoring responses for the injected value in to known HTML enabled fields. - -## Remediation - -Cross-site scripting (XSS) is an attack technique that involves echoing attacker-supplied code into a user's browser instance. A browser instance can be a standard web browser client, or a browser object embedded in a software product such as the browser within WinAmp, an RSS reader, or an email client. The code itself is usually written in HTML/JavaScript, but may also extend to VBScript, ActiveX, Java, Flash, or any other browser-supported technology. - -When an attacker gets a user's browser to execute his/her code, the code will run within the security context (or zone) of the hosting web site. With this level of privilege, the code has the ability to read, modify and transmit any sensitive data accessible by the browser. A Cross-site Scripted user could have his/her account hijacked (cookie theft), their browser redirected to another location, or possibly shown fraudulent content delivered by the web site they are visiting. Cross-site Scripting attacks essentially compromise the trust relationship between a user and the web site. Applications utilizing browser object instances which load content from the file system may execute code under the local machine zone allowing for system compromise. - -There are three types of Cross-site Scripting attacks: non-persistent, persistent and DOM-based. - -Non-persistent attacks and DOM-based attacks require a user to either visit a specially crafted link laced with malicious code, or visit a malicious web page containing a web form, which when posted to the vulnerable site, will mount the attack. Using a malicious form will oftentimes take place when the vulnerable resource only accepts HTTP POST requests. In such a case, the form can be submitted automatically, without the victim's knowledge (e.g. by using JavaScript). Upon clicking on the malicious link or submitting the malicious form, the XSS payload will get echoed back and will get interpreted by the user's browser and execute. Another technique to send almost arbitrary requests (GET and POST) is by using an embedded client, such as Adobe Flash. - -Persistent attacks occur when the malicious code is submitted to a web site where it's stored for a period of time. Examples of an attacker's favorite targets often include message board posts, web mail messages, and web chat software. The unsuspecting user is not required to interact with any additional site/link (e.g. an attacker site or a malicious link sent via email), just simply view the web page containing the code. - -## Links - -- [OWASP](https://owasp.org/Top10/A03_2021-Injection) -- [CWE](https://cwe.mitre.org/data/definitions/79.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/index.md b/doc/user/application_security/dast_api/checks/index.md index 4275b85aeb7..76387970e0c 100644 --- a/doc/user/application_security/dast_api/checks/index.md +++ b/doc/user/application_security/dast_api/checks/index.md @@ -1,140 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/index.md' +remove_date: '2024-07-30' --- -# API security testing vulnerability checks +This document was moved to [another location](../../api_security_testing/checks/index.md). -DETAILS: -**Tier:** Ultimate -**Offering:** GitLab.com, Self-managed, GitLab Dedicated - -> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/457449) from **DAST API vulnerability checks** to **API security testing vulnerability checks** in GitLab 17.0. - -[API security testing](../index.md) provides vulnerability checks that are used to -scan for vulnerabilities in the API under test. - -## Passive checks - -| Check | Severity | Type | Profiles | -|:-----------------------------------------------------------------------------|:---------|:--------|:---------| -| [Application information check](application_information_check.md) | Medium | Passive | Passive, Passive-Quick, Active-Quick, Active Full, Quick, Full | -| [Cleartext authentication check](cleartext_authentication_check.md) | High | Passive | Passive, Passive-Quick, Active-Quick, Active Full, Quick, Full | -| [JSON hijacking](json_hijacking_check.md) | Medium | Passive | Passive, Passive-Quick, Active-Quick, Active Full, Quick, Full | -| [Sensitive information](sensitive_information_disclosure_check.md) | High | Passive | Passive, Passive-Quick, Active-Quick, Active Full, Quick, Full | -| [Session cookie](session_cookie_check.md) | Medium | Passive | Passive, Passive-Quick, Active-Quick, Active Full, Quick, Full | - -## Active checks - -| Check | Severity | Type | Profiles | -|:-----------------------------------------------------------------------------|:---------|:--------|:---------| -| [CORS](cors_check.md) | Medium | Active | Active Full, Full | -| [DNS rebinding](dns_rebinding_check.md) | Medium | Active | Active Full, Full | -| [Framework debug mode](framework_debug_mode_check.md) | High | Active | Active-Quick, Active Full, Quick, Full | -| [Heartbleed OpenSSL vulnerability](heartbleed_open_ssl_check.md) | High | Active | Active Full, Full | -| [HTML injection check](html_injection_check.md) | Medium | Active | Active-Quick, Active Full, Quick, Full | -| [Insecure HTTP methods](insecure_http_methods_check.md) | Medium | Active | Active-Quick, Active Full, Quick, Full | -| [JSON injection](json_injection_check.md) | Medium | Active | Active-Quick, Active Full, Quick, Full | -| [Open redirect](open_redirect_check.md) | Medium | Active | Active Full, Full | -| [OS command injection](os_command_injection_check.md) | High | Active | Active-Quick, Active Full, Quick, Full | -| [Path traversal](path_traversal_check.md) | High | Active | Active Full, Full | -| [Sensitive file](sensitive_file_disclosure_check.md) | Medium | Active | Active Full, Full | -| [Shellshock](shellshock_check.md) | High | Active | Active Full, Full | -| [SQL injection](sql_injection_check.md) | High | Active | Active-Quick, Active Full, Quick, Full | -| [TLS configuration](tls_server_configuration_check.md) | High | Active | Active Full, Full | -| [Authentication token](authentication_token_check.md) | High | Active | Active-Quick, Active Full, Quick, Full | -| [XML external entity](xml_external_entity.md) | High | Active | Active Full, Full | -| [XML injection](xml_injection_check.md) | Medium | Active | Active-Quick, Active Full, Quick, Full | - -## API security testing checks by profile - -### Passive-Quick - -- [Application information check](application_information_check.md) -- [Cleartext authentication check](cleartext_authentication_check.md) -- [JSON hijacking](json_hijacking_check.md) -- [Sensitive information](sensitive_information_disclosure_check.md) -- [Session cookie](session_cookie_check.md) - -### Active-Quick - -- [Application information check](application_information_check.md) -- [Cleartext authentication check](cleartext_authentication_check.md) -- [Framework debug mode](framework_debug_mode_check.md) -- [HTML injection check](html_injection_check.md) -- [Insecure HTTP methods](insecure_http_methods_check.md) -- [JSON hijacking](json_hijacking_check.md) -- [JSON injection](json_injection_check.md) -- [OS command injection](os_command_injection_check.md) -- [Sensitive information](sensitive_information_disclosure_check.md) -- [Session cookie](session_cookie_check.md) -- [SQL injection](sql_injection_check.md) -- [Authentication token](authentication_token_check.md) -- [XML injection](xml_injection_check.md) - -### Active-Full - -- [Application information check](application_information_check.md) -- [Cleartext authentication check](cleartext_authentication_check.md) -- [CORS](cors_check.md) -- [DNS rebinding](dns_rebinding_check.md) -- [Framework debug mode](framework_debug_mode_check.md) -- [Heartbleed OpenSSL vulnerability](heartbleed_open_ssl_check.md) -- [HTML injection check](html_injection_check.md) -- [Insecure HTTP methods](insecure_http_methods_check.md) -- [JSON hijacking](json_hijacking_check.md) -- [JSON injection](json_injection_check.md) -- [Open redirect](open_redirect_check.md) -- [OS command injection](os_command_injection_check.md) -- [Path traversal](path_traversal_check.md) -- [Sensitive file](sensitive_file_disclosure_check.md) -- [Sensitive information](sensitive_information_disclosure_check.md) -- [Session cookie](session_cookie_check.md) -- [Shellshock](shellshock_check.md) -- [SQL injection](sql_injection_check.md) -- [TLS configuration](tls_server_configuration_check.md) -- [Authentication token](authentication_token_check.md) -- [XML injection](xml_injection_check.md) -- [XML external entity](xml_external_entity.md) - -### Quick - -- [Application information check](application_information_check.md) -- [Cleartext authentication check](cleartext_authentication_check.md) -- [Framework debug mode](framework_debug_mode_check.md) -- [HTML injection check](html_injection_check.md) -- [Insecure HTTP methods](insecure_http_methods_check.md) -- [JSON hijacking](json_hijacking_check.md) -- [JSON injection](json_injection_check.md) -- [OS command injection](os_command_injection_check.md) -- [Sensitive information](sensitive_information_disclosure_check.md) -- [Session cookie](session_cookie_check.md) -- [SQL injection](sql_injection_check.md) -- [Authentication token](authentication_token_check.md) -- [XML injection](xml_injection_check.md) - -### Full - -- [Application information check](application_information_check.md) -- [Cleartext authentication check](cleartext_authentication_check.md) -- [CORS](cors_check.md) -- [DNS rebinding](dns_rebinding_check.md) -- [Framework debug mode](framework_debug_mode_check.md) -- [Heartbleed OpenSSL vulnerability](heartbleed_open_ssl_check.md) -- [HTML injection check](html_injection_check.md) -- [Insecure HTTP methods](insecure_http_methods_check.md) -- [JSON hijacking](json_hijacking_check.md) -- [JSON injection](json_injection_check.md) -- [Open redirect](open_redirect_check.md) -- [OS command injection](os_command_injection_check.md) -- [Path traversal](path_traversal_check.md) -- [Sensitive file](sensitive_file_disclosure_check.md) -- [Sensitive information](sensitive_information_disclosure_check.md) -- [Session cookie](session_cookie_check.md) -- [Shellshock](shellshock_check.md) -- [SQL injection](sql_injection_check.md) -- [TLS configuration](tls_server_configuration_check.md) -- [Authentication token](authentication_token_check.md) -- [XML injection](xml_injection_check.md) -- [XML external entity](xml_external_entity.md) + + + + diff --git a/doc/user/application_security/dast_api/checks/insecure_http_methods_check.md b/doc/user/application_security/dast_api/checks/insecure_http_methods_check.md index aabcef34752..a8e4dda88f4 100644 --- a/doc/user/application_security/dast_api/checks/insecure_http_methods_check.md +++ b/doc/user/application_security/dast_api/checks/insecure_http_methods_check.md @@ -1,22 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/insecure_http_methods_check.md' +remove_date: '2024-07-30' --- -# Insecure HTTP methods +This document was moved to [another location](../../api_security_testing/configuration/insecure_http_methods_check.md). -## Description - -Checks to see if HTTP methods like OPTIONS and TRACE are enabled on any target endpoints. - -## Remediation - -The resource tested supports the OPTIONS HTTP method. Normally this is considered a security miss configuration as it leaks supported HTTP methods leading to information gathering about a specific server or resource. However, there is a sub-set of the API community looking to use OPTIONS as a method to self discover resource operations. If this is the intended use for enabling OPTIONS, than this issue can be considered a false positive. - -The resource tested supports the TRACE HTTP method. In combination with other cross-domain vulnerabilities in web browsers, sensitive information can be leaked from headers. It's recommended the TRACE method be disabled in your server/framework. - -## Links - -- [OWASP](https://owasp.org/Top10/A05_2021-Security_Misconfiguration) -- [CWE](https://cwe.mitre.org/data/definitions/200.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/json_hijacking_check.md b/doc/user/application_security/dast_api/checks/json_hijacking_check.md index af07dd83ecb..e8b7ec487a4 100644 --- a/doc/user/application_security/dast_api/checks/json_hijacking_check.md +++ b/doc/user/application_security/dast_api/checks/json_hijacking_check.md @@ -1,20 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/json_hijacking_check.md' +remove_date: '2024-07-30' --- -# JSON hijacking +This document was moved to [another location](../../api_security_testing/checks/json_hijacking_check.md). -## Description - -Checks for JSON data potentially vulnerable to hijacking. This check looks for a GET request that returns a JSON array, which could potentially be hijacked and read by a malicious website. - -## Remediation - -JSON hijacking allows an attacker to send a GET request via a malicious web site or similar attack vector and utilize a user's stored credentials to retrieve sensitive or protected data to which that user has access. Since a JSON array on its own is valid JavaScript, a malicious GET request to a resource that returns only a JavaScript array can allow the attacker to use a malicious script to read the data in the array from the request. GET requests should never return a JSON array, even if the resource requires authentication to access. Consider using POST instead of a GET for this request or wrapping the array in a JSON object. - -## Links - -- [OWASP](https://owasp.org/Top10/A01_2021-Broken_Access_Control) -- [CWE](https://cwe.mitre.org/data/definitions/352.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/json_injection_check.md b/doc/user/application_security/dast_api/checks/json_injection_check.md index 264d77586a4..f2959f31eb6 100644 --- a/doc/user/application_security/dast_api/checks/json_injection_check.md +++ b/doc/user/application_security/dast_api/checks/json_injection_check.md @@ -1,20 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/json_injection_check.md' +remove_date: '2024-07-30' --- -# JSON injection +This document was moved to [another location](../../api_security_testing/checks/json_injection_check.md). -## Description - -Check for JSON serialization/injection vulnerabilities. - -## Remediation - -JSON injection is an attack technique used to manipulate or compromise the logic of a JSON application or service. The injection of unintended JSON content and/or structures into an JSON message can alter the intend logic of the application. Further, JSON injection can cause the insertion of malicious content into the resulting message/document. - -## Links - -- [OWASP](https://owasp.org/Top10/A03_2021-Injection) -- [CWE](CWE-929: OWASP Top Ten 2013 Category A1 - Injection) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/open_redirect_check.md b/doc/user/application_security/dast_api/checks/open_redirect_check.md index d1670645cea..b0ffea59687 100644 --- a/doc/user/application_security/dast_api/checks/open_redirect_check.md +++ b/doc/user/application_security/dast_api/checks/open_redirect_check.md @@ -1,20 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/open_redirect_check.md' +remove_date: '2024-07-30' --- -# Open redirect +This document was moved to [another location](../../api_security_testing/checks/open_redirect_check.md). -## Description - -Identify open redirects and determine if they can be abused by attackers. - -## Remediation - -Unvalidated redirects and forwards are possible when a web application accepts untrusted input that could cause the web application to redirect the request to a URL contained within untrusted input. By modifying untrusted URL input to a malicious site, an attacker may successfully launch a phishing scam and steal user credentials. Because the server name in the modified link is identical to the original site, phishing attempts may have a more trustworthy appearance. Unvalidated redirect and forward attacks can also be used to maliciously craft a URL that would pass the application’s access control check and then forward the attacker to privileged functions that they would normally not be able to access. - -## Links - -- [OWASP](https://owasp.org/Top10/A01_2021-Broken_Access_Control) -- [CWE](https://cwe.mitre.org/data/definitions/601.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/os_command_injection_check.md b/doc/user/application_security/dast_api/checks/os_command_injection_check.md index ea399236a36..f26cba3e303 100644 --- a/doc/user/application_security/dast_api/checks/os_command_injection_check.md +++ b/doc/user/application_security/dast_api/checks/os_command_injection_check.md @@ -1,32 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/os_command_injection_check.md' +remove_date: '2024-07-30' --- -# OS command injection +This document was moved to [another location](../../api_security_testing/checks/os_command_injection_check.md). -## Description - -Check for OS command injection vulnerabilities. An OS command injection attack consists of insertion or "injection" of an OS command via the input data from the client to the application. -A successful OS command injection exploit can run arbitrary commands. This allows an attacker the ability to read, write, and delete data. Depending on the user the commands run as, this can also include administrative functions. - -This check modifies parameters in the request (path, query string, headers, JSON, XML, etc.) to try and execute an OS command. Both standard injections and blind injections are performed. Blind injections cause delays in response when successful. - -## Remediation - -It is possible to execute arbitrary OS commands on the target application server. OS Command Injection is a critical vulnerability that can lead to a full system compromise. User input should never be used in constructing commands or command arguments to functions which execute OS commands. This includes filenames supplied by user uploads or downloads. - -Ensure your application does not: - -- Use user supplied information in the process name to execute. -- Use user supplied information in an OS command execution function which does -not escape shell meta-characters. -- Use user supplied information in arguments to OS commands. - -The application should have a hardcoded set of arguments that are to be passed to OS commands. If filenames are being passed to these functions, it is recommended that a hash of the filename be used instead, or some other unique identifier. It is strongly recommended that a native library that implements the same functionality be used instead of using OS system commands due to the risk of unknown attacks against third party commands. - -## Links - -- [OWASP](https://owasp.org/Top10/A03_2021-Injection) -- [CWE](https://cwe.mitre.org/data/definitions/78.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/path_traversal_check.md b/doc/user/application_security/dast_api/checks/path_traversal_check.md index 3fa5c16b184..4e81ece2bca 100644 --- a/doc/user/application_security/dast_api/checks/path_traversal_check.md +++ b/doc/user/application_security/dast_api/checks/path_traversal_check.md @@ -1,30 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/path_traversal_check.md' +remove_date: '2024-07-30' --- -# Path traversal +This document was moved to [another location](../../api_security_testing/checks/path_traversal_check.md). -## Description - -Many file operations are intended to take place within a restricted directory. By using special elements such as ".." and "/" separators, attackers can escape outside of the restricted location to access files or directories that are elsewhere on the system. One of the most common special elements is the "../" sequence, which in most modern operating systems is interpreted as the parent directory of the current location. This is referred to as relative path traversal. Path traversal also covers the use of absolute path-names such as "/usr/local/bin", which may also be useful in accessing unexpected files. This is referred to as absolute path traversal. - -In many programming languages, the injection of a null byte(the 0 or NUL) may allow an attacker to truncate a generated filename to widen the scope of attack. For example, the software may add ".txt" to any pathname, thus limiting the attacker to text files, but a null injection may effectively remove this restriction. - -This check modifies parameters in the request (path, query string, headers, JSON, XML, etc.) to try and access restricted files and files outside of the web-root. Logs and responses are then analyzed to try and detect if the file was successfully accessed. - -## Remediation - -The Path traversal attack technique allows an attacker access to files, directories, and commands that potentially reside outside the web document root directory. An attacker may manipulate a URL in such a way that the web site will execute or reveal the contents of arbitrary files anywhere on the web server. Any device that exposes an HTTP-based interface is potentially vulnerable to Path traversal. - -Most web sites restrict user access to a specific portion of the file-system, typically called the “web document root” or “CGI root” directory. These directories contain the files intended for user access and the executable necessary to drive web application functionality. To access files or execute commands anywhere on the file-system, Path traversal attacks will utilize the ability of special-characters sequences. - -The most basic Path traversal attack uses the “../” special-character sequence to alter the resource location requested in the URL. Although most popular web servers will prevent this technique from escaping the web document root, alternate encodings of the "../" sequence may help bypass the security filters. These method variations include valid and invalid Unicode-encoding ("..%u2216" or "..%c0%af") of the forward slash character, backslash characters ("..") on Windows-based servers, URL encoded characters "%2e%2e%2f"), and double URL encoding ("..%255c") of the backslash character. - -Even if the web server properly restricts Path traversal attempts in the URL path, a web application itself may still be vulnerable due to improper handling of user-supplied input. This is a common problem of web applications that use template mechanisms or load static text from files. In variations of the attack, the original URL parameter value is substituted with the file name of one of the web application's dynamic scripts. Consequently, the results can reveal source code because the file is interpreted as text instead of an executable script. These techniques often employ additional special characters such as the dot (".") to reveal the listing of the current working directory, or “%00” NULL characters in order to bypass rudimentary file extension checks. - -## Links - -- [OWASP](https://owasp.org/Top10/A01_2021-Broken_Access_Control) -- [CWE](https://cwe.mitre.org/data/definitions/22.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/sensitive_file_disclosure_check.md b/doc/user/application_security/dast_api/checks/sensitive_file_disclosure_check.md index 5c2f9ca0723..b48f02771ea 100644 --- a/doc/user/application_security/dast_api/checks/sensitive_file_disclosure_check.md +++ b/doc/user/application_security/dast_api/checks/sensitive_file_disclosure_check.md @@ -1,22 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/sensitive_file_disclosure_check.md' +remove_date: '2024-07-30' --- -# Sensitive file disclosure +This document was moved to [another location](../../api_security_testing/checks/sensitive_file_disclosure_check.md). -## Description - -Check for sensitive file disclosure. This check looks for files that may contain sensitive information. Examples include .htaccess, .htpasswd, .bash_history, etc. - -## Remediation - -Information leakage is an application weakness where an application reveals sensitive data, such as technical details of the web application, environment, or user-specific data. Sensitive data may be used by an attacker to exploit the target web application, its hosting network, or its users. Therefore, leakage of sensitive data should be limited or prevented whenever possible. Information Leakage, in its most common form,is the result of one or more of the following conditions: A failure to scrub out HTML/Script comments containing sensitive information, improper application or server configurations, or differences in page responses for valid versus invalid data. - -In the case of this failure, one or more files and/or folders are accessable that should not be. This can include files common in home folders like such as command histories or files that contain secrets such as passwords. - -## Links - -- [OWASP](https://owasp.org/Top10/A01_2021-Broken_Access_Control) -- [CWE](https://cwe.mitre.org/data/definitions/200.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/sensitive_information_disclosure_check.md b/doc/user/application_security/dast_api/checks/sensitive_information_disclosure_check.md index 7ab04cda52c..05511df4daf 100644 --- a/doc/user/application_security/dast_api/checks/sensitive_information_disclosure_check.md +++ b/doc/user/application_security/dast_api/checks/sensitive_information_disclosure_check.md @@ -1,22 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/sensitive_information_disclosure_check.md' +remove_date: '2024-07-30' --- -# Sensitive information disclosure +This document was moved to [another location](../../api_security_testing/configuration/sensitive_information_disclosure_check.md). -## Description - -Sensitive information disclosure check. This includes credit card numbers, health records, personal information, etc. - -## Remediation - -Sensitive information leakage is an application weakness where an application reveals sensitive, user-specific data. Sensitive data may be used by an attacker to exploit its users. Therefore, leakage of sensitive data should be limited or prevented whenever possible. Information Leakage, in its most common form, is the result of differences in page responses for valid versus invalid data. - -Pages that provide different responses based on the validity of the data can also lead to Information Leakage; specifically when data deemed confidential is being revealed as a result of the web application's design. Examples of sensitive data includes (but is not limited to): account numbers, user identifiers (Drivers license number, Passport number, Social Security Numbers, etc.) and user-specific information (passwords, sessions, addresses). Information Leakage in this context deals with exposure of key user data deemed confidential, or secret, that should not be exposed in plain view, even to the user. Credit card numbers and other heavily regulated information are prime examples of user data that needs to be further protected from exposure or leakage even with proper encryption and access controls already in place. - -## Links - -- [OWASP](https://owasp.org/Top10/A01_2021-Broken_Access_Control) -- [CWE](https://cwe.mitre.org/data/definitions/200.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/session_cookie_check.md b/doc/user/application_security/dast_api/checks/session_cookie_check.md index 0f4def9f74d..084243a4979 100644 --- a/doc/user/application_security/dast_api/checks/session_cookie_check.md +++ b/doc/user/application_security/dast_api/checks/session_cookie_check.md @@ -1,27 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/session_cookie_check.md' +remove_date: '2024-07-30' --- -# Session cookie +This document was moved to [another location](../../api_security_testing/checks/session_cookie_check.md). -## Description - -Verify session cookie has correct flags and expiration. - -## Remediation - -Since HTTP is a stateless protocol, web sites commonly use cookies to store session IDs that uniquely identify a user from request to request. Consequently, each session ID's confidentiality must be maintained in order to prevent multiple users from accessing the same account. A stolen session ID can be used to view another user's account or perform a fraudulent -transaction. - -- One part of securing session ID's is to property mark them to expire and also require the correct set of flags to ensure they are not transmitted in the clear or accessible from scripting. -- HttpOnly is an additional flag included in a Set-Cookie HTTP response header. Using the HttpOnly flag when generating a cookie helps mitigate the risk of client side script accessing the protected cookie (if the browser supports it). If the HttpOnly flag (optional) is included in the HTTP response header, the cookie cannot be accessed through client side script (again if the browser supports this flag). As a result, even if a cross-site scripting (XSS) flaw exists, and a user accidentally accesses a link that exploits this flaw, the browser will not reveal the cookie to a third party. -- The Secure attribute for sensitive cookies in HTTPS sessions is not set, which could cause the user agent to send those cookies in plaintext over an HTTP session. -- A session related cookie was identified being used on an insecure transport protocol. Insecure transport protocols are those that do not make use of SSL/TLS to secure the connection. Examples of such protocols are 'http'. -- Insufficient Session Expiration occurs when a Web application permits an attacker to reuse old session credentials or session IDs for authorization. Insufficient Session Expiration increases a website's exposure to attacks that steal or reuse user's session identifiers. Since HTTP is a stateless protocol, websites commonly use cookies to store session IDs that uniquely identify a user from request to request. Consequently, each session ID's confidentiality must be maintained in order to prevent multiple users from accessing the same account. A stolen session ID can be used to view another user's account or perform a fraudulent transaction. One part of securing session ID's is to property mark them to expire and also require the correct set of flags to ensure they are not transmitted in the clear or accessible from scripting. - -## Links - -- [OWASP](https://owasp.org/Top10/A07_2021-Identification_and_Authentication_Failures) -- [CWE](https://cwe.mitre.org/data/definitions/930.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/shellshock_check.md b/doc/user/application_security/dast_api/checks/shellshock_check.md index 314554005f4..3ee4d8ef25d 100644 --- a/doc/user/application_security/dast_api/checks/shellshock_check.md +++ b/doc/user/application_security/dast_api/checks/shellshock_check.md @@ -1,20 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/shellshock_check.md' +remove_date: '2024-07-30' --- -# Shellshock +This document was moved to [another location](../../api_security_testing/checks/shellshock_check.md). -## Description - -Check for Shellshock vulnerabilities. - -## Remediation - -Shellshock vulnerability takes advantage of a bug in BASH, in which, BASH incorrectly executes trailing commands when it imports a function definition stored into an environment variable. Any environment which allows defining BASH environmental variables could be vulnerable to this bug, as for example a Apache Web Server using mod_cgi and mod_cgid modules. A known-good request was modified to include malicious content. The malicious content includes an Shell shock attack in which the server-side application returns a specific text (evidence) in the response headers. - -## Links - -- [OWASP](https://owasp.org/Top10/A03_2021-Injection) -- [CWE](https://cwe.mitre.org/data/definitions/78.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/sql_injection_check.md b/doc/user/application_security/dast_api/checks/sql_injection_check.md index c4b7671bdf9..5bc88d3e267 100644 --- a/doc/user/application_security/dast_api/checks/sql_injection_check.md +++ b/doc/user/application_security/dast_api/checks/sql_injection_check.md @@ -1,24 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/sql_injection_check.md' +remove_date: '2024-07-30' --- -# SQL injection +This document was moved to [another location](../../api_security_testing/checks/sql_injection_check.md). -## Description - -Check for SQL and NoSQL injection vulnerabilities. A SQL injection attack consists of insertion or "injection" of a SQL query via the input data from the client to the application. A successful SQL injection exploit can read sensitive data from the database, modify database data (Insert/Update/Delete), execute administration operations on the database (such as shutdown the DBMS), recover the content of a given file present on the DBMS file system and in some cases issue commands to the operating system. SQL injection attacks are a type of injection attack, in which SQL commands are injected into data-plane input in order to effect the execution of predefined SQL commands. This check modifies parameters in the request (path, query string, headers, JSON, XML, etc.) to try and create a syntax error in the SQL or NoSQL query. Logs and responses are then analyzed to try and detect if an error occured. If an error is detected there is a high likelihood that a vulnerability exists. - -## Remediation - -The software constructs all or part of an SQL command using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the intended SQL command when it is sent to a downstream component. - -Without sufficient removal or quoting of SQL syntax in user-controllable inputs, the generated SQL query can cause those inputs to be interpreted as SQL instead of ordinary user data. This can be used to alter query logic to bypass security checks, or to insert additional statements that modify the back-end database, possibly including execution of system commands. - -SQL injection has become a common issue with database-driven websites. The flaw is easily detected, and easily exploited, and as such, any site or software package with even a minimal user base is likely to be subject to an attempted attack of this kind. This flaw depends on the fact that SQL makes no real distinction between the control and data planes. - -## Links - -- [OWASP](https://owasp.org/Top10/A03_2021-Injection) -- [CWE](https://cwe.mitre.org/data/definitions/930.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/tls_server_configuration_check.md b/doc/user/application_security/dast_api/checks/tls_server_configuration_check.md index b7183ba177d..16fb92b79c7 100644 --- a/doc/user/application_security/dast_api/checks/tls_server_configuration_check.md +++ b/doc/user/application_security/dast_api/checks/tls_server_configuration_check.md @@ -1,29 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/tls_server_configuration_check.md' +remove_date: '2024-07-30' --- -# TLS server configuration +This document was moved to [another location](../../api_security_testing/checks/tls_server_configuration_check.md). -## Description - -Check for various TLS Server configuration issues. Checks TLS versions, hmacs, ciphers and compression algs supported by server. - -## Remediation - -Insufficient transport layer protection allows communication to be exposed to untrusted third-parties, providing an attack vector to compromise a web application and/or steal sensitive information. Websites typically use Secure Sockets Layer/Transport Layer Security (SSL/TLS) to provide encryption at the transport layer. However, unless the website is configured to use SSL/TLS and configured to use SSL/TLS properly, the website may be vulnerable to traffic interception and modification. - -SSL/TLS as a protocol have gone through several revisions over the years. Each new version adds features and fixes weaknesses in the protocol. Over time some versions of the protocol are broken so badly as to become vulnerabilities if supported. It's recommended to support only the most recent TLS versions such as TLS 1.3 (2018), and TLS 1.2 (2008). - -Compression has been linked to side-channel attacks on TLS connections. Disabling compression can prevent these attacks. One attack in particular, CRIME ("Compression Ratio Info-leak Made Easy") can be prevented. CRIME is an attack that targets clients, but if the server does not support compression the attack is mitigated. - -Historically, high grade cryptography was restricted from export to outside the United States. Because of this, websites were configured to support weak cryptographic options for those clients that were restricted to only using weak ciphers. Weak ciphers are vulnerable to attack because of the relative ease of breaking them; less than two weeks on a typical home computer -and a few seconds using dedicated hardware. - -Today, all modern browsers and websites use much stronger encryption, but some websites are still configured to support outdated weak ciphers. Because of this, an attacker may be able to force the client to downgrade to a weaker cipher when connecting to the website, allowing the attacker to break the weak encryption. For this reason, the server should be configured to only accept strong ciphers and not provide service to any client that requests using a weaker cipher. In addition, some websites are misconfigured to choose a weaker cipher even when the client will support a much stronger one. OWASP offers a guide to testing for SSL/TLS issues, including weak cipher support and misconfiguration, and there are other resources and tools as well. - -## Links - -- [OWASP](https://owasp.org/Top10/A02_2021-Cryptographic_Failures) -- [CWE](https://cwe.mitre.org/data/definitions/934.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/xml_external_entity.md b/doc/user/application_security/dast_api/checks/xml_external_entity.md index e86076a9316..21299fa9414 100644 --- a/doc/user/application_security/dast_api/checks/xml_external_entity.md +++ b/doc/user/application_security/dast_api/checks/xml_external_entity.md @@ -1,20 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/xml_external_entity_check.md' +remove_date: '2024-07-30' --- -# XML external entity +This document was moved to [another location](../../api_security_testing/checks/xml_external_entity_check.md). -## Description - -Check for XML DTD processing vulnerabilities. - -## Remediation - -XML external entity Attack is a type of attack against an application that parses XML input. This attack occurs when XML input containing a reference to an external entity is processed by a weakly configured XML parser. This attack may lead to the disclosure of confidential data, denial of service, server side request forgery, port scanning from the perspective of the machine where the parser is located, and other system impacts. - -## Links - -- [OWASP](https://owasp.org/Top10/A03_2021-Injection) -- [CWE](https://cwe.mitre.org/data/definitions/611.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/checks/xml_injection_check.md b/doc/user/application_security/dast_api/checks/xml_injection_check.md index d1bb4c6a8c9..8c8fd66c778 100644 --- a/doc/user/application_security/dast_api/checks/xml_injection_check.md +++ b/doc/user/application_security/dast_api/checks/xml_injection_check.md @@ -1,20 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/checks/xml_injection_check.md' +remove_date: '2024-07-30' --- -# XML Injection Check +This document was moved to [another location](../../api_security_testing/checks/xml_injection_check.md). -## Description - -Check for XML serialization/injection vulnerabilities. - -## Remediation - -XML Injection is an attack technique used to manipulate or compromise the logic of an XML application or service. The injection of unintended XML content and/or structures into an XML message can alter the intend logic of the application. Further, XML injection can cause the insertion of malicious content into the resulting message/document. - -## Links - -- [OWASP](https://owasp.org/Top10/A03_2021-Injection) -- [CWE](https://cwe.mitre.org/data/definitions/91.html) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/configuration/customizing_analyzer_settings.md b/doc/user/application_security/dast_api/configuration/customizing_analyzer_settings.md index e63a02ac605..913ec0856ae 100644 --- a/doc/user/application_security/dast_api/configuration/customizing_analyzer_settings.md +++ b/doc/user/application_security/dast_api/configuration/customizing_analyzer_settings.md @@ -1,1014 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/configuration/customizing_analyzer_settings.md' +remove_date: '2024-07-30' --- -# Customizing analyzer settings +This document was moved to [another location](../../api_security_testing/configuration/customizing_analyzer_settings.md). -## Authentication - -Authentication is handled by providing the authentication token as a header or cookie. You can -provide a script that performs an authentication flow or calculates the token. - -### HTTP Basic Authentication - -[HTTP basic authentication](https://en.wikipedia.org/wiki/Basic_access_authentication) -is an authentication method built into the HTTP protocol and used in conjunction with -[transport layer security (TLS)](https://en.wikipedia.org/wiki/Transport_Layer_Security). - -We recommended that you [create a CI/CD variable](../../../../ci/variables/index.md#for-a-project) -for the password (for example, `TEST_API_PASSWORD`), and set it to be masked. You can create CI/CD -variables from the GitLab project's page at **Settings > CI/CD**, in the **Variables** section. -Because of the [limitations on masked variables](../../../../ci/variables/index.md#mask-a-cicd-variable), -you should Base64-encode the password before adding it as a variable. - -Finally, add two CI/CD variables to your `.gitlab-ci.yml` file: - -- `DAST_API_HTTP_USERNAME`: The username for authentication. -- `DAST_API_HTTP_PASSWORD_BASE64`: The Base64-encoded password for authentication. - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_HAR: test-api-recording.har - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_HTTP_USERNAME: testuser - DAST_API_HTTP_PASSWORD_BASE64: $TEST_API_PASSWORD -``` - -#### Raw password - -If you do not want to Base64-encode the password (or if you are using GitLab 15.3 or earlier) you can provide the raw password `DAST_API_HTTP_PASSWORD`, instead of using `DAST_API_HTTP_PASSWORD_BASE64`. - -### Bearer tokens - -Bearer tokens are used by several different authentication mechanisms, including OAuth2 and JSON Web -Tokens (JWT). Bearer tokens are transmitted using the `Authorization` HTTP header. To use Bearer -tokens with API security testing, you need one of the following: - -- A token that doesn't expire. -- A way to generate a token that lasts the length of testing. -- A Python script that API security testing can call to generate the token. - -#### Token doesn't expire - -If the Bearer token doesn't expire, use the `DAST_API_OVERRIDES_ENV` variable to provide it. This -variable's content is a JSON snippet that provides headers and cookies to add to outgoing HTTP requests for API security testing. - -Follow these steps to provide the Bearer token with `DAST_API_OVERRIDES_ENV`: - -1. [Create a CI/CD variable](../../../../ci/variables/index.md#for-a-project), - for example `TEST_API_BEARERAUTH`, with the value - `{"headers":{"Authorization":"Bearer dXNlcm5hbWU6cGFzc3dvcmQ="}}` (substitute your token). You - can create CI/CD variables from the GitLab projects page at **Settings > CI/CD**, in the - **Variables** section. - Due to the format of `TEST_API_BEARERAUTH` it's not possible to mask the variable. - To mask the token's value, you can create a second variable with the token values, and define - `TEST_API_BEARERAUTH` with the value `{"headers":{"Authorization":"Bearer $MASKED_VARIABLE"}}`. - -1. In your `.gitlab-ci.yml` file, set `DAST_API_OVERRIDES_ENV` to the variable you just created: - - ```yaml - stages: - - dast - - include: - - template: DAST-API.gitlab-ci.yml - - variables: - DAST_API_PROFILE: Quick - DAST_API_OPENAPI: test-api-specification.json - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_OVERRIDES_ENV: $TEST_API_BEARERAUTH - ``` - -1. To validate that authentication is working, run API security testing and review the job logs - and the test API's application logs. - -#### Token generated at test runtime - -If the Bearer token must be generated and doesn't expire during testing, you can provide API security testing with a file that has the token. A prior stage and job, or part of the API security testing job, can -generate this file. - -API security testing expects to receive a JSON file with the following structure: - -```json -{ - "headers" : { - "Authorization" : "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" - } -} -``` - -This file can be generated by a prior stage and provided to API security testing through the -`DAST_API_OVERRIDES_FILE` CI/CD variable. - -Set `DAST_API_OVERRIDES_FILE` in your `.gitlab-ci.yml` file: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_OPENAPI: test-api-specification.json - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_OVERRIDES_FILE: dast-api-overrides.json -``` - -To validate that authentication is working, run API security testing and review the job logs and -the test API's application logs. - -#### Token has short expiration - -If the Bearer token must be generated and expires prior to the scan's completion, you can provide a -program or script for the API security testing scanner to execute on a provided interval. The provided script runs in -an Alpine Linux container that has Python 3 and Bash installed. If the Python script requires -additional packages, it must detect this and install the packages at runtime. - -The script must create a JSON file containing the Bearer token in a specific format: - -```json -{ - "headers" : { - "Authorization" : "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" - } -} -``` - -You must provide three CI/CD variables, each set for correct operation: - -- `DAST_API_OVERRIDES_FILE`: JSON file the provided command generates. -- `DAST_API_OVERRIDES_CMD`: Command that generates the JSON file. -- `DAST_API_OVERRIDES_INTERVAL`: Interval (in seconds) to run command. - -For example: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_OPENAPI: test-api-specification.json - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_OVERRIDES_FILE: dast-api-overrides.json - DAST_API_OVERRIDES_CMD: renew_token.py - DAST_API_OVERRIDES_INTERVAL: 300 -``` - -To validate that authentication is working, run API security testing and review the job logs and the test API's application logs. See the [overrides section](#overrides) for more information about override commands. - -## Overrides - -API security testing provides a method to add or override specific items in your request, for example: - -- Headers -- Cookies -- Query string -- Form data -- JSON nodes -- XML nodes - -You can use this to inject semantic version headers, authentication, and so on. The -[authentication section](#authentication) includes examples of using overrides for that purpose. - -Overrides use a JSON document, where each type of override is represented by a JSON object: - -```json -{ - "headers": { - "header1": "value", - "header2": "value" - }, - "cookies": { - "cookie1": "value", - "cookie2": "value" - }, - "query": { - "query-string1": "value", - "query-string2": "value" - }, - "body-form": { - "form-param1": "value", - "form-param2": "value" - }, - "body-json": { - "json-path1": "value", - "json-path2": "value" - }, - "body-xml" : { - "xpath1": "value", - "xpath2": "value" - } -} -``` - -Example of setting a single header: - -```json -{ - "headers": { - "Authorization": "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" - } -} -``` - -Example of setting both a header and cookie: - -```json -{ - "headers": { - "Authorization": "Bearer dXNlcm5hbWU6cGFzc3dvcmQ=" - }, - "cookies": { - "flags": "677" - } -} -``` - -Example usage for setting a `body-form` override: - -```json -{ - "body-form": { - "username": "john.doe" - } -} -``` - -The override engine uses `body-form` when the request body has only form-data content. - -Example usage for setting a `body-json` override: - -```json -{ - "body-json": { - "$.credentials.access-token": "iddqd!42.$" - } -} -``` - -Each JSON property name in the object `body-json` is set to a [JSON Path](https://goessner.net/articles/JsonPath/) -expression. The JSON Path expression `$.credentials.access-token` identifies the node to be -overridden with the value `iddqd!42.$`. The override engine uses `body-json` when the request body -has only [JSON](https://www.json.org/json-en.html) content. - -For example, if the body is set to the following JSON: - -```json -{ - "credentials" : { - "username" :"john.doe", - "access-token" : "non-valid-password" - } -} -``` - -It is changed to: - -```json -{ - "credentials" : { - "username" :"john.doe", - "access-token" : "iddqd!42.$" - } -} -``` - -Here's an example for setting a `body-xml` override. The first entry overrides an XML attribute and -the second entry overrides an XML element: - -```json -{ - "body-xml" : { - "/credentials/@isEnabled": "true", - "/credentials/access-token/text()" : "iddqd!42.$" - } -} -``` - -Each JSON property name in the object `body-xml` is set to an -[XPath v2](https://www.w3.org/TR/xpath20/) -expression. The XPath expression `/credentials/@isEnabled` identifies the attribute node to override -with the value `true`. The XPath expression `/credentials/access-token/text()` identifies the -element node to override with the value `iddqd!42.$`. The override engine uses `body-xml` when the -request body has only [XML](https://www.w3.org/XML/) -content. - -For example, if the body is set to the following XML: - -```xml - - john.doe - non-valid-password - -``` - -It is changed to: - -```xml - - john.doe - iddqd!42.$ - -``` - -You can provide this JSON document as a file or environment variable. You may also provide a command -to generate the JSON document. The command can run at intervals to support values that expire. - -### Using a file - -To provide the overrides JSON as a file, the `DAST_API_OVERRIDES_FILE` CI/CD variable is set. The path is relative to the job current working directory. - -Here's an example `.gitlab-ci.yml`: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_OPENAPI: test-api-specification.json - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_OVERRIDES_FILE: dast-api-overrides.json -``` - -### Using a CI/CD variable - -To provide the overrides JSON as a CI/CD variable, use the `DAST_API_OVERRIDES_ENV` variable. -This allows you to place the JSON as variables that can be masked and protected. - -In this example `.gitlab-ci.yml`, the `DAST_API_OVERRIDES_ENV` variable is set directly to the JSON: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_OPENAPI: test-api-specification.json - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_OVERRIDES_ENV: '{"headers":{"X-API-Version":"2"}}' -``` - -In this example `.gitlab-ci.yml`, the `SECRET_OVERRIDES` variable provides the JSON. This is a -[group or instance level CI/CD variable defined in the UI](../../../../ci/variables/index.md#define-a-cicd-variable-in-the-ui): - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_OPENAPI: test-api-specification.json - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_OVERRIDES_ENV: $SECRET_OVERRIDES -``` - -### Using a command - -If the value must be generated or regenerated on expiration, you can provide a program or script for -the API security testing scanner to execute on a specified interval. The provided command runs in an Alpine Linux -container that has Python 3 and Bash installed. - -You have to set the environment variable `DAST_API_OVERRIDES_CMD` to the program or script you would like -to execute. The provided command creates the overrides JSON file as defined previously. - -You might want to install other scripting runtimes like NodeJS or Ruby, or maybe you need to install a dependency for -your overrides command. In this case, we recommend setting the `DAST_API_PRE_SCRIPT` to the file path of a script which -provides those prerequisites. The script provided by `DAST_API_PRE_SCRIPT` is executed once, before the analyzer starts. - -See the [Alpine Linux package management](https://wiki.alpinelinux.org/wiki/Alpine_Linux_package_management) -page for information about installing Alpine Linux packages. - -You must provide three CI/CD variables, each set for correct operation: - -- `DAST_API_OVERRIDES_FILE`: File generated by the provided command. -- `DAST_API_OVERRIDES_CMD`: Overrides command in charge of generating the overrides JSON file periodically. -- `DAST_API_OVERRIDES_INTERVAL`: Interval in seconds to run command. - -Optionally: - -- `DAST_API_PRE_SCRIPT`: Script to install runtimes or dependencies before the scan starts. - -WARNING: -To execute scripts in Alpine Linux you must first use the command [`chmod`](https://www.gnu.org/software/coreutils/manual/html_node/chmod-invocation.html) to set the [execution permission](https://www.gnu.org/software/coreutils/manual/html_node/Setting-Permissions.html). For example, to set the execution permission of `script.py` for everyone, use the command: `chmod a+x script.py`. If needed, you can version your `script.py` with the execution permission already set. - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_OPENAPI: test-api-specification.json - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_OVERRIDES_FILE: dast-api-overrides.json - DAST_API_OVERRIDES_CMD: renew_token.py - DAST_API_OVERRIDES_INTERVAL: 300 -``` - -### Debugging overrides - -By default the output of the overrides command is hidden. If the overrides command returns a non zero exit code, the command is displayed as part of your job output. Optionally, you can set the variable `DAST_API_OVERRIDES_CMD_VERBOSE` to any value to display overrides command output as it is generated. This is useful when testing your overrides script, but should be disabled afterwards as it slows down testing. - -It is also possible to write messages from your script to a log file that is collected when the job completes or fails. The log file must be created in a specific location and following a naming convention. - -Adding some basic logging to your overrides script is useful in case the script fails unexpectedly during standard running of the job. The log file is automatically included as an artifact of the job, allowing you to download it after the job has finished. - -Following our example, we provided `renew_token.py` in the environment variable `DAST_API_OVERRIDES_CMD`. Notice two things in the script: - -- Log file is saved in the location indicated by the environmental variable `CI_PROJECT_DIR`. -- Log filename should match `gl-*.log`. - -```python -#!/usr/bin/env python - -# Example of an overrides command - -# Override commands can update the overrides json file -# with new values to be used. This is a great way to -# update an authentication token that will expire -# during testing. - -import logging -import json -import os -import requests -import backoff - -# [1] Store log file in directory indicated by env var CI_PROJECT_DIR -working_directory = os.environ.get( 'CI_PROJECT_DIR') -overrides_file_name = os.environ.get('DAST_API_OVERRIDES_FILE', 'dast-api-overrides.json') -overrides_file_path = os.path.join(working_directory, overrides_file_name) - -# [2] File name should match the pattern: gl-*.log -log_file_path = os.path.join(working_directory, 'gl-user-overrides.log') - -# Set up logger -logging.basicConfig(filename=log_file_path, level=logging.DEBUG) - -# Use `backoff` decorator to retry in case of transient errors. -@backoff.on_exception(backoff.expo, - (requests.exceptions.Timeout, - requests.exceptions.ConnectionError), - max_time=30) -def get_auth_response(): - authorization_url = 'https://authorization.service/api/get_api_token' - return requests.get( - f'{authorization_url}', - auth=(os.environ.get('AUTH_USER'), os.environ.get('AUTH_PWD')) - ) - -# In our example, access token is retrieved from a given endpoint -try: - - # Performs a http request, response sample: - # { "Token" : "abcdefghijklmn" } - response = get_auth_response() - - # Check that the request is successful. may raise `requests.exceptions.HTTPError` - response.raise_for_status() - - # Gets JSON data - response_body = response.json() - -# If needed specific exceptions can be caught -# requests.ConnectionError : A network connection error problem occurred -# requests.HTTPError : HTTP request returned an unsuccessful status code. [Response.raise_for_status()] -# requests.ConnectTimeout : The request timed out while trying to connect to the remote server -# requests.ReadTimeout : The server did not send any data in the allotted amount of time. -# requests.TooManyRedirects : The request exceeds the configured number of maximum redirections -# requests.exceptions.RequestException : All exceptions that related to Requests -except json.JSONDecodeError as json_decode_error: - # logs errors related decoding JSON response - logging.error(f'Error, failed while decoding JSON response. Error message: {json_decode_error}') - raise -except requests.exceptions.RequestException as requests_error: - # logs exceptions related to `Requests` - logging.error(f'Error, failed while performing HTTP request. Error message: {requests_error}') - raise -except Exception as e: - # logs any other error - logging.error(f'Error, unknown error while retrieving access token. Error message: {e}') - raise - -# computes object that holds overrides file content. -# It uses data fetched from request -overrides_data = { - "headers": { - "Authorization": f"Token {response_body['Token']}" - } -} - -# log entry informing about the file override computation -logging.info("Creating overrides file: %s" % overrides_file_path) - -# attempts to overwrite the file -try: - if os.path.exists(overrides_file_path): - os.unlink(overrides_file_path) - - # overwrites the file with our updated dictionary - with open(overrides_file_path, "wb+") as fd: - fd.write(json.dumps(overrides_data).encode('utf-8')) -except Exception as e: - # logs any other error - logging.error(f'Error, unknown error when overwriting file {overrides_file_path}. Error message: {e}') - raise - -# logs informing override has finished successfully -logging.info("Override file has been updated") - -# end -``` - -In the overrides command example, the Python script depends on the `backoff` library. To make sure the library is installed before executing the Python script, the `DAST_API_PRE_SCRIPT` is set to a script that installs the dependencies of your overrides command. -As for example, the following script `user-pre-scan-set-up.sh` - -```shell -#!/bin/bash - -# user-pre-scan-set-up.sh -# Ensures python dependencies are installed - -echo "**** install python dependencies ****" - -python3 -m ensurepip -pip3 install --no-cache --upgrade \ - pip \ - backoff - -echo "**** python dependencies installed ****" - -# end -``` - -You have to update your configuration to set the `DAST_API_PRE_SCRIPT` to our new `user-pre-scan-set-up.sh` script. For example: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_OPENAPI: test-api-specification.json - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_PRE_SCRIPT: user-pre-scan-set-up.sh - DAST_API_OVERRIDES_FILE: dast-api-overrides.json - DAST_API_OVERRIDES_CMD: renew_token.py - DAST_API_OVERRIDES_INTERVAL: 300 -``` - -In the previous sample, you could use the script `user-pre-scan-set-up.sh` to also install new runtimes or applications that later on you could use in our overrides command. - -## Request Headers - -The request headers feature lets you specify fixed values for the headers during the scan session. For example, you can use the configuration variable `DAST_API_REQUEST_HEADERS` to set a fixed value in the `Cache-Control` header. If the headers you need to set include sensitive values like the `Authorization` header, use the [masked variable](../../../../ci/variables/index.md#mask-a-cicd-variable) feature along with the [variable `DAST_API_REQUEST_HEADERS_BASE64`](#base64). - -If the `Authorization` header or any other header needs to get updated while the scan is in progress, consider using the [overrides](#overrides) feature. - -The variable `DAST_API_REQUEST_HEADERS` lets you specify a comma-separated (`,`) list of headers. These headers are included on each request that the scanner performs. Each header entry in the list consists of a name followed by a colon (`:`) and then by its value. Whitespace before the key or value is ignored. For example, to declare a header name `Cache-Control` with the value `max-age=604800`, the header entry is `Cache-Control: max-age=604800`. To use two headers, `Cache-Control: max-age=604800` and `Age: 100`, set `DAST_API_REQUEST_HEADERS` variable to `Cache-Control: max-age=604800, Age: 100`. - -The order in which the different headers are provided into the variable `DAST_API_REQUEST_HEADERS` does not affect the result. Setting `DAST_API_REQUEST_HEADERS` to `Cache-Control: max-age=604800, Age: 100` produces the same result as setting it to `Age: 100, Cache-Control: max-age=604800`. - -### Base64 - -The `DAST_API_REQUEST_HEADERS_BASE64` variable accepts the same list of headers as `DAST_API_REQUEST_HEADERS`, with the only difference that the entire value of the variable must be Base64-encoded. For example, to set `DAST_API_REQUEST_HEADERS_BASE64` variable to `Authorization: QmVhcmVyIFRPS0VO, Cache-control: bm8tY2FjaGU=`, ensure you convert the list to its Base64 equivalent: `QXV0aG9yaXphdGlvbjogUW1WaGNtVnlJRlJQUzBWTywgQ2FjaGUtY29udHJvbDogYm04dFkyRmphR1U9`, and the Base64-encoded value must be used. This is useful when storing secret header values in a [masked variable](../../../../ci/variables/index.md#mask-a-cicd-variable), which has character set restrictions. - -WARNING: -Base64 is used to support the [masked variable](../../../../ci/variables/index.md#mask-a-cicd-variable) feature. Base64 encoding is not by itself a security measure, because sensitive values can be easily decoded. - -### Example: Adding a list of headers on each request using plain text - -In the following example of a `.gitlab-ci.yml`, `DAST_API_REQUEST_HEADERS` configuration variable is set to provide two header values as explained in [request headers](#request-headers). - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_OPENAPI: test-api-specification.json - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_REQUEST_HEADERS: 'Cache-control: no-cache, Save-Data: on' -``` - -### Example: Using a masked CI/CD variable - -The following `.gitlab-ci.yml` sample assumes the [masked variable](../../../../ci/variables/index.md#mask-a-cicd-variable) `SECRET_REQUEST_HEADERS_BASE64` is defined as a [group or instance level CI/CD variable defined in the UI](../../../../ci/variables/index.md#define-a-cicd-variable-in-the-ui). The value of `SECRET_REQUEST_HEADERS_BASE64` is set to `WC1BQ01FLVNlY3JldDogc31jcnt0ISwgWC1BQ01FLVRva2VuOiA3MDVkMTZmNWUzZmI=`, which is the Base64-encoded text version of `X-ACME-Secret: s3cr3t!, X-ACME-Token: 705d16f5e3fb`. Then, it can be used as follows: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_OPENAPI: test-api-specification.json - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_REQUEST_HEADERS_BASE64: $SECRET_REQUEST_HEADERS_BASE64 -``` - -Consider using `DAST_API_REQUEST_HEADERS_BASE64` when storing secret header values in a [masked variable](../../../../ci/variables/index.md#mask-a-cicd-variable), which has character set restrictions. - -## Exclude Paths - -When testing an API it can be useful to exclude certain paths. For example, you might exclude testing of an authentication service or an older version of the API. To exclude paths, use the `DAST_API_EXCLUDE_PATHS` CI/CD variable . This variable is specified in your `.gitlab-ci.yml` file. To exclude multiple paths, separate entries using the `;` character. In the provided paths you can use a single character wildcard `?` and `*` for a multiple character wildcard. - -To verify the paths are excluded, review the `Tested Operations` and `Excluded Operations` portion of the job output. You should not see any excluded paths listed under `Tested Operations`. - -```plaintext -2021-05-27 21:51:08 [INF] DAST API: --[ Tested Operations ]------------------------- -2021-05-27 21:51:08 [INF] DAST API: 201 POST http://target:7777/api/users CREATED -2021-05-27 21:51:08 [INF] DAST API: ------------------------------------------------ -2021-05-27 21:51:08 [INF] DAST API: --[ Excluded Operations ]----------------------- -2021-05-27 21:51:08 [INF] DAST API: GET http://target:7777/api/messages -2021-05-27 21:51:08 [INF] DAST API: POST http://target:7777/api/messages -2021-05-27 21:51:08 [INF] DAST API: ------------------------------------------------ -``` - -### Examples - -This example excludes the `/auth` resource. This does not exclude child resources (`/auth/child`). - -```yaml -variables: - DAST_API_EXCLUDE_PATHS: /auth -``` - -To exclude `/auth`, and child resources (`/auth/child`), we use a wildcard. - -```yaml -variables: - DAST_API_EXCLUDE_PATHS: /auth* -``` - -To exclude multiple paths we use the `;` character. In this example we exclude `/auth*` and `/v1/*`. - -```yaml -variables: - DAST_API_EXCLUDE_PATHS: /auth*;/v1/* -``` - -To exclude one or more nested levels within a path we use `**`. In this example we are testing API endpoints. We are testing `/api/v1/` and `/api/v2/` of a data query requesting `mass`, `brightness` and `coordinates` data for `planet`, `moon`, `star`, and `satellite` objects. Example paths that could be scanned include, but are not limited to: - -- `/api/v2/planet/coordinates` -- `/api/v1/star/mass` -- `/api/v2/satellite/brightness` - -In this example we test the `brightness` endpoint only: - -```yaml -variables: - DAST_API_EXCLUDE_PATHS: /api/**/mass;/api/**/coordinates -``` - -### Exclude parameters - -While testing an API you may might want to exclude a parameter (query string, header, or body element) from testing. This may be needed because a parameter always causes a failure, slows down testing, or for other reasons. To exclude parameters, you can set one of the following variables: `DAST_API_EXCLUDE_PARAMETER_ENV` or `DAST_API_EXCLUDE_PARAMETER_FILE`. - -The `DAST_API_EXCLUDE_PARAMETER_ENV` allows providing a JSON string containing excluded parameters. This is a good option if the JSON is short and does not change often. Another option is the variable `DAST_API_EXCLUDE_PARAMETER_FILE`. This variable is set to a file path that can be checked into the repository, created by another job as an artifact, or generated at runtime with a pre-script using `DAST_API_PRE_SCRIPT`. - -#### Exclude parameters using a JSON document - -The JSON document contains a JSON object, this object uses specific properties to identify which parameter should be excluded. -You can provide the following properties to exclude specific parameters during the scanning process: - -- `headers`: Use this property to exclude specific headers. The property's value is an array of header names to be excluded. Names are case-insensitive. -- `cookies`: Use this property's value to exclude specific cookies. The property's value is an array of cookie names to be excluded. Names are case-sensitive. -- `query`: Use this property to exclude specific fields from the query string. The property's value is an array of field names from the query string to be excluded. Names are case-sensitive. -- `body-form`: Use this property to exclude specific fields from a request that uses the media type `application/x-www-form-urlencoded`. The property's value is an array of the field names from the body to be excluded. Names are case-sensitive. -- `body-json`: Use this property to exclude specific JSON nodes from a request that uses the media type `application/json`. The property's value is an array, each entry of the array is a [JSON Path](https://goessner.net/articles/JsonPath/) expression. -- `body-xml`: Use this property to exclude specific XML nodes from a request that uses media type `application/xml`. The property's value is an array, each entry of the array is a [XPath v2](https://www.w3.org/TR/xpath20/) expression. - -Thus, the following JSON document is an example of the expected structure to exclude parameters. - -```json -{ - "headers": [ - "header1", - "header2" - ], - "cookies": [ - "cookie1", - "cookie2" - ], - "query": [ - "query-string1", - "query-string2" - ], - "body-form": [ - "form-param1", - "form-param2" - ], - "body-json": [ - "json-path-expression-1", - "json-path-expression-2" - ], - "body-xml" : [ - "xpath-expression-1", - "xpath-expression-2" - ] -} -``` - -#### Examples - -##### Excluding a single header - -To exclude the header `Upgrade-Insecure-Requests`, set the `header` property's value to an array with the header name: `[ "Upgrade-Insecure-Requests" ]`. For instance, the JSON document looks like this: - -```json -{ - "headers": [ "Upgrade-Insecure-Requests" ] -} -``` - -Header names are case-insensitive, so the header name `UPGRADE-INSECURE-REQUESTS` is equivalent to `Upgrade-Insecure-Requests`. - -##### Excluding both a header and two cookies - -To exclude the header `Authorization`, and the cookies `PHPSESSID` and `csrftoken`, set the `headers` property's value to an array with header name `[ "Authorization" ]` and the `cookies` property's value to an array with the cookies' names `[ "PHPSESSID", "csrftoken" ]`. For instance, the JSON document looks like this: - -```json -{ - "headers": [ "Authorization" ], - "cookies": [ "PHPSESSID", "csrftoken" ] -} -``` - -##### Excluding a `body-form` parameter - -To exclude the `password` field in a request that uses `application/x-www-form-urlencoded`, set the `body-form` property's value to an array with the field name `[ "password" ]`. For instance, the JSON document looks like this: - -```json -{ - "body-form": [ "password" ] -} -``` - -The exclude parameters uses `body-form` when the request uses a content type `application/x-www-form-urlencoded`. - -##### Excluding a specific JSON nodes using JSON Path - -To exclude the `schema` property in the root object, set the `body-json` property's value to an array with the JSON Path expression `[ "$.schema" ]`. - -The JSON Path expression uses special syntax to identify JSON nodes: `$` refers to the root of the JSON document, `.` refers to the current object (in our case the root object), and the text `schema` refers to a property name. Thus, the JSON path expression `$.schema` refers to a property `schema` in the root object. -For instance, the JSON document looks like this: - -```json -{ - "body-json": [ "$.schema" ] -} -``` - -The exclude parameters uses `body-json` when the request uses a content type `application/json`. Each entry in `body-json` is expected to be a [JSON Path expression](https://goessner.net/articles/JsonPath/). In JSON Path characters like `$`, `*`, `.` among others have special meaning. - -##### Excluding multiple JSON nodes using JSON Path - -To exclude the property `password` on each entry of an array of `users` at the root level, set the `body-json` property's value to an array with the JSON Path expression `[ "$.users[*].paswword" ]`. - -The JSON Path expression starts with `$` to refer to the root node and uses `.` to refer to the current node. Then, it uses `users` to refer to a property and the characters `[` and `]` to enclose the index in the array you want to use, instead of providing a number as an index you use `*` to specify any index. After the index reference, we find `.` which now refers to any given selected index in the array, preceded by a property name `password`. - -For instance, the JSON document looks like this: - -```json -{ - "body-json": [ "$.users[*].paswword" ] -} -``` - -The exclude parameters uses `body-json` when the request uses a content type `application/json`. Each entry in `body-json` is expected to be a [JSON Path expression](https://goessner.net/articles/JsonPath/). In JSON Path characters like `$`, `*`, `.` among others have special meaning. - -##### Excluding a XML attribute - -To exclude an attribute named `isEnabled` located in the root element `credentials`, set the `body-xml` property's value to an array with the XPath expression `[ "/credentials/@isEnabled" ]`. - -The XPath expression `/credentials/@isEnabled`, starts with `/` to indicate the root of the XML document, then it is followed by the word `credentials` which indicates the name of the element to match. It uses a `/` to refer to a node of the previous XML element, and the character `@` to indicate that the name `isEnable` is an attribute. - -For instance, the JSON document looks like this: - -```json -{ - "body-xml": [ - "/credentials/@isEnabled" - ] -} -``` - -The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. - -##### Excluding a XML text's element - -To exclude the text of the `username` element contained in root node `credentials`, set the `body-xml` property's value to an array with the XPath expression `[/credentials/username/text()" ]`. - -In the XPath expression `/credentials/username/text()`, the first character `/` refers to the root XML node, and then after it indicates an XML element's name `credentials`. Similarly, the character `/` refers to the current element, followed by a new XML element's name `username`. Last part has a `/` that refers to the current element, and uses a XPath function called `text()` which identifies the text of the current element. - -For instance, the JSON document looks like this: - -```json -{ - "body-xml": [ - "/credentials/username/text()" - ] -} -``` - -The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. - -##### Excluding an XML element - -To exclude the element `username` contained in root node `credentials`, set the `body-xml` property's value to an array with the XPath expression `[/credentials/username" ]`. - -In the XPath expression `/credentials/username`, the first character `/` refers to the root XML node, and then after it indicates an XML element's name `credentials`. Similarly, the character `/` refers to the current element, followed by a new XML element's name `username`. - -For instance, the JSON document looks like this: - -```json -{ - "body-xml": [ - "/credentials/username" - ] -} -``` - -The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. - -##### Excluding an XML node with namespaces - -To exclude anXML element `login` which is defined in namespace `s`, and contained in `credentials` root node, set the `body-xml` property's value to an array with the XPath expression `[ "/credentials/s:login" ]`. - -In the XPath expression `/credentials/s:login`, the first character `/` refers to the root XML node, and then after it indicates an XML element's name `credentials`. Similarly, the character `/` refers to the current element, followed by a new XML element's name `s:login`. Notice that name contains the character `:`, this character separates the namespace from the node name. - -The namespace name should have been defined in the XML document which is part of the body request. You may check the namespace in the specification document HAR, OpenAPI, or Postman Collection file. - -```json -{ - "body-xml": [ - "/credentials/s:login" - ] -} -``` - -The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be an [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath, expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. - -#### Using a JSON string - -To provide the exclusion JSON document set the variable `DAST_API_EXCLUDE_PARAMETER_ENV` with the JSON string. In the following example, the `.gitlab-ci.yml`, the `DAST_API_EXCLUDE_PARAMETER_ENV` variable is set to a JSON string: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_OPENAPI: test-api-specification.json - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_EXCLUDE_PARAMETER_ENV: '{ "headers": [ "Upgrade-Insecure-Requests" ] }' -``` - -#### Using a file - -To provide the exclusion JSON document set the variable `DAST_API_EXCLUDE_PARAMETER_FILE` with the JSON file path. The file path is relative to the job current working directory. In the following example `.gitlab-ci.yml` content, the `DAST_API_EXCLUDE_PARAMETER_FILE` variable is set to a JSON file path: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_OPENAPI: test-api-specification.json - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_EXCLUDE_PARAMETER_FILE: dast-api-exclude-parameters.json -``` - -The `dast-api-exclude-parameters.json` is a JSON document that follows the structure of [exclude parameters document](#exclude-parameters-using-a-json-document). - -### Exclude URLs - -As an alternative to excluding by paths, you can filter by any other component in the URL by using the `DAST_API_EXCLUDE_URLS` CI/CD variable. This variable can be set in your `.gitlab-ci.yml` file. The variable can store multiple values, separated by commas (`,`). Each value is a regular expression. Because each entry is a regular expression, an entry like `.*` excludes all URLs because it is a regular expression that matches everything. - -In your job output you can check if any URLs matched any provided regular expression from `DAST_API_EXCLUDE_URLS`. Matching operations are listed in the **Excluded Operations** section. Operations listed in the **Excluded Operations** should not be listed in the **Tested Operations** section. For example the following portion of a job output: - -```plaintext -2021-05-27 21:51:08 [INF] DAST API: --[ Tested Operations ]------------------------- -2021-05-27 21:51:08 [INF] DAST API: 201 POST http://target:7777/api/users CREATED -2021-05-27 21:51:08 [INF] DAST API: ------------------------------------------------ -2021-05-27 21:51:08 [INF] DAST API: --[ Excluded Operations ]----------------------- -2021-05-27 21:51:08 [INF] DAST API: GET http://target:7777/api/messages -2021-05-27 21:51:08 [INF] DAST API: POST http://target:7777/api/messages -2021-05-27 21:51:08 [INF] DAST API: ------------------------------------------------ -``` - -NOTE: -Each value in `DAST_API_EXCLUDE_URLS` is a regular expression. Characters such as `.` , `*` and `$` among many others have special meanings in [regular expressions](https://en.wikipedia.org/wiki/Regular_expression#Standards). - -#### Examples - -##### Excluding a URL and child resources - -The following example excludes the URL `http://target/api/auth` and its child resources. - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_TARGET_URL: http://target/ - DAST_API_OPENAPI: test-api-specification.json - DAST_API_EXCLUDE_URLS: http://target/api/auth -``` - -##### Excluding two URLs and allow their child resources - -To exclude the URLs `http://target/api/buy` and `http://target/api/sell` but allowing to scan their child resources, for instance: `http://target/api/buy/toy` or `http://target/api/sell/chair`. You could use the value `http://target/api/buy/$,http://target/api/sell/$`. This value is using two regular expressions, each of them separated by a `,` character. Hence, it contains `http://target/api/buy$` and `http://target/api/sell$`. In each regular expression, the trailing `$` character points out where the matching URL should end. - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_TARGET_URL: http://target/ - DAST_API_OPENAPI: test-api-specification.json - DAST_API_EXCLUDE_URLS: http://target/api/buy/$,http://target/api/sell/$ -``` - -##### Excluding two URLs and their child resources - -To exclude the URLs: `http://target/api/buy` and `http://target/api/sell`, and their child resources. To provide multiple URLs we use the `,` character as follows: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_TARGET_URL: http://target/ - DAST_API_OPENAPI: test-api-specification.json - DAST_API_EXCLUDE_URLS: http://target/api/buy,http://target/api/sell -``` - -##### Excluding URL using regular expressions - -To exclude exactly `https://target/api/v1/user/create` and `https://target/api/v2/user/create` or any other version (`v3`,`v4`, and more). We could use `https://target/api/v.*/user/create$`, in the previous regular expression `.` indicates any character and `*` indicates zero or more times, additionally `$` indicates that the URL should end there. - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_TARGET_URL: http://target/ - DAST_API_OPENAPI: test-api-specification.json - DAST_API_EXCLUDE_URLS: https://target/api/v.*/user/create$ -``` + + + + diff --git a/doc/user/application_security/dast_api/configuration/enabling_the_analyzer.md b/doc/user/application_security/dast_api/configuration/enabling_the_analyzer.md index 2107d862bfd..f8f92af549e 100644 --- a/doc/user/application_security/dast_api/configuration/enabling_the_analyzer.md +++ b/doc/user/application_security/dast_api/configuration/enabling_the_analyzer.md @@ -1,937 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/configuration/enabling_the_analyzer.md' +remove_date: '2024-07-30' --- -# Enabling the analyzer +This document was moved to [another location](../../api_security_testing/configuration/enabling_the_analyzer.md). -You can specify the API you want to scan by using: - -- [OpenAPI v2 or v3 Specification](#openapi-specification) -- [GraphQL Schema](#graphql-schema) -- [HTTP Archive (HAR)](#http-archive-har) -- [Postman Collection v2.0 or v2.1](#postman-collection) - -## OpenAPI Specification - -The [OpenAPI Specification](https://www.openapis.org/) (formerly the Swagger Specification) is an API description format for REST APIs. -This section shows you how to configure API security testing scanning using an OpenAPI Specification to provide information about the target API to test. -OpenAPI Specifications are provided as a file system resource or URL. Both JSON and YAML OpenAPI formats are supported. - -API security testing uses an OpenAPI document to generate the request body. When a request body is required, -the body generation is limited to these body types: - -- `application/x-www-form-urlencoded` -- `multipart/form-data` -- `application/json` -- `application/xml` - -## OpenAPI and media types - -A media type (formerly known as MIME type) is an identifier for file formats and format contents transmitted. A OpenAPI document lets you specify that a given operation can accept different media types, hence a given request can send data using different file content. As for example, a `PUT /user` operation to update user data could accept data in either XML (media type `application/xml`) or JSON (media type `application/json`) format. -OpenAPI 2.x lets you specify the accepted media types globally or per operation, and OpenAPI 3.x lets you specify the accepted media types per operation. API security testing will check the listed media types, and try to produce sample data for each supported media type. - -- The default behavior is to select one of the supported media types to use. The first supported media type is chosen from the list. This behavior is configurable. - -Testing the same operation (for example, `POST /user`) using different media types (for example, `application/json` and `application/xml`) is not always desirable. -For example, if the target application executes the same code regardless of the request content type, it will take longer to finish the test session, and it may report duplicated vulnerabilities related to the request body depending on the target app. - -The environment variable `DAST_API_OPENAPI_ALL_MEDIA_TYPES` lets you specify whether or not to use all supported media types instead of one when generating requests for a given operation. When the environment variable `DAST_API_OPENAPI_ALL_MEDIA_TYPES` is set to any value, API security testing tries to generate requests for all supported media types instead of one in a given operation. This will cause testing to take longer as testing is repeated for each provided media type. - -Alternatively, the variable `DAST_API_OPENAPI_MEDIA_TYPES` is used to provide a list of media types that will each be tested. Providing more than one media type causes testing to take longer, as testing is performed for each media type selected. When the environment variable `DAST_API_OPENAPI_MEDIA_TYPES` is set to a list of media types, only the listed media types are included when creating requests. - -Multiple media types in `DAST_API_OPENAPI_MEDIA_TYPES` are separated by a colon (`:`). For example, to limit request generation to the media types `application/x-www-form-urlencoded` and `multipart/form-data`, set the environment variable `DAST_API_OPENAPI_MEDIA_TYPES` to `application/x-www-form-urlencoded:multipart/form-data`. Only supported media types in this list are included when creating requests, though non-supported media types are always skipped. A media type text may contain different sections. For example, `application/vnd.api+json; charset=UTF-8`, is a compound of `type "/" [tree "."] subtype ["+" suffix]* [";" parameter]`. Parameters are not taken into account when performing the filtering media types on request generation. - -The environment variables `DAST_API_OPENAPI_ALL_MEDIA_TYPES` and `DAST_API_OPENAPI_MEDIA_TYPES` allow you to decide how to handle media types. These settings are mutually exclusive. If both are enabled, API security testing reports an error. - -### Configure API security testing with an OpenAPI Specification - -To configure API security testing scanning with an OpenAPI Specification: - -1. [Include](../../../../ci/yaml/index.md#includetemplate) - the [`DAST-API.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml) in your `.gitlab-ci.yml` file. - -1. The [configuration file](variables.md#configuration-files) has several testing profiles defined with different checks enabled. We recommend that you start with the `Quick` profile. - Testing with this profile completes faster, allowing for easier configuration validation. - Provide the profile by adding the `DAST_API_PROFILE` CI/CD variable to your `.gitlab-ci.yml` file. - -1. Provide the location of the OpenAPI Specification as either a file or URL. - Specify the location by adding the `DAST_API_OPENAPI` variable. - -1. The target API instance's base URL is also required. Provide it by using the `DAST_API_TARGET_URL` - variable or an `environment_url.txt` file. - - Adding the URL in an `environment_url.txt` file at your project's root is great for testing in - dynamic environments. To run API security testing against an app dynamically created during a GitLab CI/CD - pipeline, have the app persist its URL in an `environment_url.txt` file. API security testing - automatically parses that file to find its scan target. You can see an - [example of this in our Auto DevOps CI YAML](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml). - -Complete example configuration of using an OpenAPI Specification: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_OPENAPI: test-api-specification.json - DAST_API_TARGET_URL: http://test-deployment/ -``` - -This is a minimal configuration for API security testing. From here you can: - -- [Run your first scan](#running-your-first-scan). -- [Add authentication](customizing_analyzer_settings.md#authentication). -- Learn how to [handle false positives](#handling-false-positives). - -## HTTP Archive (HAR) - -The [HTTP Archive format (HAR)](../../api_fuzzing/create_har_files.md) is an archive file format for -logging HTTP transactions. When used with the GitLab API security testing scanner, the HAR file must contain -records of calling the web API to test. The API security testing scanner extracts all of the requests and uses them -to perform testing. - -You can use various tools to generate HAR files: - -- [Insomnia Core](https://insomnia.rest/): API client -- [Chrome](https://www.google.com/chrome/): Browser -- [Firefox](https://www.mozilla.org/en-US/firefox/): Browser -- [Fiddler](https://www.telerik.com/fiddler): Web debugging proxy -- [GitLab HAR Recorder](https://gitlab.com/gitlab-org/security-products/har-recorder): Command line - -WARNING: -HAR files may contain sensitive information such as authentication tokens, API keys, and session -cookies. We recommend that you review the HAR file contents before adding them to a repository. - -### API security testing scanning with a HAR file - -To configure API security testing to use a HAR file that provides information about the target API to test: - -1. [Include](../../../../ci/yaml/index.md#includetemplate) - the [`DAST-API.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml) in your `.gitlab-ci.yml` file. - -1. The [configuration file](variables.md#configuration-files) has several testing profiles defined with different checks enabled. We recommend that you start with the `Quick` profile. - Testing with this profile completes faster, allowing for easier configuration validation. - - Provide the profile by adding the `DAST_API_PROFILE` CI/CD variable to your `.gitlab-ci.yml` file. - -1. Provide the location of the HAR file. You can provide the location as a file path - or URL. Specify the location by adding the `DAST_API_HAR` variable. - -1. The target API instance's base URL is also required. Provide it by using the `DAST_API_TARGET_URL` - variable or an `environment_url.txt` file. - - Adding the URL in an `environment_url.txt` file at your project's root is great for testing in - dynamic environments. To run API security testing against an app dynamically created during a GitLab CI/CD - pipeline, have the app persist its URL in an `environment_url.txt` file. API security testing - automatically parses that file to find its scan target. You can see an - [example of this in our Auto DevOps CI YAML](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml). - -Complete example configuration of using an HAR file: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_HAR: test-api-recording.har - DAST_API_TARGET_URL: http://test-deployment/ -``` - -This example is a minimal configuration for API security testing. From here you can: - -- [Run your first scan](#running-your-first-scan). -- [Add authentication](customizing_analyzer_settings.md#authentication). -- Learn how to [handle false positives](#handling-false-positives). - -## GraphQL Schema - -> - Support for GraphQL Schema was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352780) in GitLab 15.4. - -GraphQL is a query language for your API and an alternative to REST APIs. -API security testing supports testing GraphQL endpoints multiple ways: - -- Test using the GraphQL Schema. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352780) in GitLab 15.4. -- Test using a recording (HAR) of GraphQL queries. -- Test using a Postman Collection containing GraphQL queries. - -This section documents how to test using a GraphQL schema. The GraphQL schema support in -API security testing is able to query the schema from endpoints that support [introspection](https://graphql.org/learn/introspection/). -Introspection is enabled by default to allow tools like GraphiQL to work. -For details on how to enable introspection, see your GraphQL framework documentation. - -### API security testing scanning with a GraphQL endpoint URL - -The GraphQL support in API security testing is able to query a GraphQL endpoint for the schema. - -NOTE: -The GraphQL endpoint must support introspection queries for this method to work correctly. - -To configure API security testing to use a GraphQL endpoint URL that provides information about the target API to test: - -1. [Include](../../../../ci/yaml/index.md#includetemplate) - the [`DAST-API.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml) in your `.gitlab-ci.yml` file. - -1. Provide the path to the GraphQL endpoint, for example `/api/graphql`. Specify the location by adding the `DAST_API_GRAPHQL` variable. - -1. The target API instance's base URL is also required. Provide it by using the `DAST_API_TARGET_URL` - variable or an `environment_url.txt` file. - - Adding the URL in an `environment_url.txt` file at your project's root is great for testing in - dynamic environments. See the [dynamic environment solutions](../troubleshooting.md#dynamic-environment-solutions) section of our documentation for more information. - -Complete example configuration of using a GraphQL endpoint path: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -dast_api: - variables: - DAST_API_GRAPHQL: /api/graphql - DAST_API_TARGET_URL: http://test-deployment/ -``` - -This example is a minimal configuration for API security testing. From here you can: - -- [Run your first scan](#running-your-first-scan). -- [Add authentication](customizing_analyzer_settings.md#authentication). -- Learn how to [handle false positives](#handling-false-positives). - -### API security testing scanning with a GraphQL Schema file - -API security testing can use a GraphQL schema file to understand and test a GraphQL endpoint that has introspection disabled. To use a GraphQL schema file, it must be in the introspection JSON format. A GraphQL schema can be converted to a the introspection JSON format using an online 3rd party tool: [https://transform.tools/graphql-to-introspection-json](https://transform.tools/graphql-to-introspection-json). - -To configure API security testing to use a GraphQL schema file that provides information about the target API to test: - -1. [Include](../../../../ci/yaml/index.md#includetemplate) - the [`DAST-API.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml) in your `.gitlab-ci.yml` file. - -1. Provide the GraphQL endpoint path, for example `/api/graphql`. Specify the path by adding the `DAST_API_GRAPHQL` variable. - -1. Provide the location of the GraphQL schema file. You can provide the location as a file path - or URL. Specify the location by adding the `DAST_API_GRAPHQL_SCHEMA` variable. - -1. The target API instance's base URL is also required. Provide it by using the `DAST_API_TARGET_URL` - variable or an `environment_url.txt` file. - - Adding the URL in an `environment_url.txt` file at your project's root is great for testing in - dynamic environments. See the [dynamic environment solutions](../troubleshooting.md#dynamic-environment-solutions) section of our documentation for more information. - -Complete example configuration of using an GraphQL schema file: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -dast_api: - variables: - DAST_API_GRAPHQL: /api/graphql - DAST_API_GRAPHQL_SCHEMA: test-api-graphql.schema - DAST_API_TARGET_URL: http://test-deployment/ -``` - -Complete example configuration of using an GraphQL schema file URL: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -dast_api: - variables: - DAST_API_GRAPHQL: /api/graphql - DAST_API_GRAPHQL_SCHEMA: http://file-store/files/test-api-graphql.schema - DAST_API_TARGET_URL: http://test-deployment/ -``` - -This example is a minimal configuration for API security testing. From here you can: - -- [Run your first scan](#running-your-first-scan). -- [Add authentication](customizing_analyzer_settings.md#authentication). -- Learn how to [handle false positives](#handling-false-positives). - -## Postman Collection - -The [Postman API Client](https://www.postman.com/product/api-client/) is a popular tool that -developers and testers use to call various types of APIs. The API definitions -[can be exported as a Postman Collection file](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-postman-data) -for use with API security testing. When exporting, make sure to select a supported version of Postman -Collection: v2.0 or v2.1. - -When used with the GitLab API security testing scanner, Postman Collections must contain definitions of the web API to -test with valid data. The API security testing scanner extracts all the API definitions and uses them to perform -testing. - -WARNING: -Postman Collection files may contain sensitive information such as authentication tokens, API keys, -and session cookies. We recommend that you review the Postman Collection file contents before adding -them to a repository. - -### API security testing scanning with a Postman Collection file - -To configure API security testing to use a Postman Collection file that provides information about the target -API to test: - -1. [Include](../../../../ci/yaml/index.md#includetemplate) - the [`DAST-API.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml). - -1. The [configuration file](variables.md#configuration-files) has several testing profiles defined with different checks enabled. We recommend that you start with the `Quick` profile. - Testing with this profile completes faster, allowing for easier configuration validation. - - Provide the profile by adding the `DAST_API_PROFILE` CI/CD variable to your `.gitlab-ci.yml` file. - -1. Provide the location of the Postman Collection file as either a file or URL. Specify the location by adding the `DAST_API_POSTMAN_COLLECTION` variable. - -1. The target API instance's base URL is also required. Provide it by using the `DAST_API_TARGET_URL` - variable or an `environment_url.txt` file. - - Adding the URL in an `environment_url.txt` file at your project's root is great for testing in - dynamic environments. To run API security testing against an app dynamically created during a GitLab CI/CD - pipeline, have the app persist its URL in an `environment_url.txt` file. API security testing - automatically parses that file to find its scan target. You can see an - [example of this in our Auto DevOps CI YAML](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml). - -Complete example configuration of using a Postman collection: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_POSTMAN_COLLECTION: postman-collection_serviceA.json - DAST_API_TARGET_URL: http://test-deployment/ -``` - -This is a minimal configuration for API security testing. From here you can: - -- [Run your first scan](#running-your-first-scan). -- [Add authentication](customizing_analyzer_settings.md#authentication). -- Learn how to [handle false positives](#handling-false-positives). - -### Postman variables - -> - Support for Postman Environment file format was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356312) in GitLab 15.1. -> - Support for multiple variable files was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356312) in GitLab 15.1. -> - Support for Postman variable scopes: Global and Environment was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356312) in GitLab 15.1. - -#### Variables in Postman Client - -Postman allows the developer to define placeholders that can be used in different parts of the -requests. These placeholders are called variables, as explained in [using variables](https://learning.postman.com/docs/sending-requests/variables/). -You can use variables to store and reuse values in your requests and scripts. For example, you can -edit the collection to add variables to the document: - -![Edit collection variable tab View](../img/dast_api_postman_collection_edit_variable.png) - -Or alternatively, you can add variables in an environment: - -![Edit environment variables View](../img/dast_api_postman_environment_edit_variable.png) - -You can then use the variables in sections such as URL, headers, and others: - -![Edit request using variables View](../img/dast_api_postman_request_edit.png) - -Postman has grown from a basic client tool with a nice UX experience to a more complex ecosystem that allows testing APIs with scripts, creating complex collections that trigger secondary requests, and setting variables along the way. Not every feature in the Postman ecosystem is supported. For example, scripts are not supported. The main focus of the Postman support is to ingest Postman Collection definitions that are used by the Postman Client and their related variables defined in the workspace, environments, and the collections themselves. - -Postman allows creating variables in different scopes. Each scope has a different level of visibility in the Postman tools. For example, you can create a variable in a _global environment_ scope that is seen by every operation definition and workspace. You can also create a variable in a specific _environment_ scope that is only visible and used when that specific environment is selected for use. Some scopes are not always available, for example in the Postman ecosystem you can create requests in the Postman Client, these requests do not have a _local_ scope, but test scripts do. - -Variable scopes in Postman can be a daunting topic and not everyone is familiar with it. We strongly recommend that you read [Variable Scopes](https://learning.postman.com/docs/sending-requests/variables/#variable-scopes) from Postman documentation before moving forward. - -As mentioned above, there are different variable scopes, and each of them has a purpose and can be used to provide more flexibility to your Postman document. There is an important note on how values for variables are computed, as per Postman documentation: - -> If a variable with the same name is declared in two different scopes, the value stored in the variable with narrowest scope is used. For example, if there is a global variable named `username` and a local variable named `username`, the local value is used when the request runs. - -The following is a summary of the variable scopes supported by the Postman Client and API security testing: - -- **Global Environment (Global) scope** is a special pre-defined environment that is available throughout a workspace. We can also refer to the _global environment_ scope as the _global_ scope. The Postman Client allows exporting the global environment into a JSON file, which can be used with API security testing. -- **Environment scope** is a named group of variables created by a user in the Postman Client. - The Postman Client supports a single active environment along with the global environment. The variables defined in an active user-created environment take precedence over variables defined in the global environment. The Postman Client allows exporting your environment into a JSON file, which can be used with API security testing. -- **Collection scope** is a group of variables declared in a given collection. The collection variables are available to the collection where they have been declared and the nested requests or collections. Variables defined in the collection scope take precedence over the _global environment_ scope and also the _environment_ scope. - The Postman Client can export one or more collections into a JSON file, this JSON file contains selected collections, requests, and collection variables. -- **API security testing scope** is a new scope added by API security testing to allow users to provide extra variables, or override variables defined in other supported scopes. This scope is not supported by Postman. The _API security testing scope_ variables are provided using a [custom JSON file format](#api-security-testing-scope-custom-json-file-format). - - Override values defined in the environment or collection - - Defining variables from scripts - - Define a single row of data from the unsupported _data scope_ -- **Data scope** is a group of variables in which their name and values come from JSON or CSV files. A Postman collection runner like [Newman](https://learning.postman.com/docs/running-collections/using-newman-cli/command-line-integration-with-newman/) or [Postman Collection Runner](https://learning.postman.com/docs/running-collections/intro-to-collection-runs/) executes the requests in a collection as many times as entries have the JSON or CSV file. A good use case for these variables is to automate tests using scripts in Postman. - API security testing does **not** support reading data from a CSV or JSON file. -- **Local scope** are variables that are defined in Postman scripts. API security testing does **not** support Postman scripts and by extension, variables defined in scripts. You can still provide values for the script-defined variables by defining them in one of the supported scopes, or our custom JSON format. - -Not all scopes are supported by API security testing and variables defined in scripts are not supported. The following table is sorted by broadest scope to narrowest scope. - -| Scope | Postman | API security testing | Comment | -|----------------------------|:-------:|:--------------------:|:-------------------------------------------| -| Global Environment | Yes | Yes | Special pre-defined environment | -| Environment | Yes | Yes | Named environments | -| Collection | Yes | Yes | Defined in your postman collection | -| API security testing scope | No | Yes | Custom scope added by API security testing | -| Data | Yes | No | External files in CSV or JSON format | -| Local | Yes | No | Variables defined in scripts | - -For more details on how to define variables and export variables in different scopes, see: - -- [Defining collection variables](https://learning.postman.com/docs/sending-requests/variables/#defining-collection-variables) -- [Defining environment variables](https://learning.postman.com/docs/sending-requests/variables/#defining-environment-variables) -- [Defining global variables](https://learning.postman.com/docs/sending-requests/variables/#defining-global-variables) - -##### Exporting from Postman Client - -The Postman Client lets you export different file formats, for instance, you can export a Postman collection or a Postman environment. -The exported environment can be the global environment (which is always available) or can be any custom environment you previously have created. When you export a Postman Collection, it may contain only declarations for _collection_ and _local_ scoped variables; _environment_ scoped variables are not included. - -To get the declaration for _environment_ scoped variables, you have to export a given environment at the time. Each exported file only includes variables from the selected environment. - -For more details on exporting variables in different supported scopes, see: - -- [Exporting collections](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-collections) -- [Exporting environments](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-environments) -- [Downloading global environments](https://learning.postman.com/docs/sending-requests/variables/#downloading-global-environments) - -#### API security testing scope, custom JSON file format - -Our custom JSON file format is a JSON object where each object property represents a variable name and the property value represents the variable value. This file can be created using your favorite text editor, or it can be produced by an earlier job in your pipeline. - -This example defines two variables `base_url` and `token` in the API security testing scope: - -```json -{ - "base_url": "http://127.0.0.1/", - "token": "Token 84816165151" -} -``` - -#### Using scopes with API security testing - -The scopes: _global_, _environment_, _collection_, and _GitLab API security testing_ are supported in [GitLab 15.1 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/356312). GitLab 15.0 and earlier, supports only the _collection_, and _GitLab API security testing_ scopes. - -The following table provides a quick reference for mapping scope files/URLs to API security testing configuration variables: - -| Scope | How to Provide | -| ------------------ | --------------- | -| Global environment | DAST_API_POSTMAN_COLLECTION_VARIABLES | -| Environment | DAST_API_POSTMAN_COLLECTION_VARIABLES | -| Collection | DAST_API_POSTMAN_COLLECTION | -| API security testing scope | DAST_API_POSTMAN_COLLECTION_VARIABLES | -| Data | Not supported | -| Local | Not supported | - -The Postman Collection document automatically includes any _collection_ scoped variables. The Postman Collection is provided with the configuration variable `DAST_API_POSTMAN_COLLECTION`. This variable can be set to a single [exported Postman collection](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-collections). - -Variables from other scopes are provided through the `DAST_API_POSTMAN_COLLECTION_VARIABLES` configuration variable. The configuration variable supports a comma (`,`) delimited file list in [GitLab 15.1 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/356312). GitLab 15.0 and earlier, supports only one single file. The order of the files provided is not important as the files provide the needed scope information. - -The configuration variable `DAST_API_POSTMAN_COLLECTION_VARIABLES` can be set to: - -- [Exported Global environment](https://learning.postman.com/docs/sending-requests/variables/#downloading-global-environments) -- [Exported environments](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-environments) -- [API security testing Custom JSON format](#api-security-testing-scope-custom-json-file-format) - -#### Undefined Postman variables - -There is a chance that API security testing engine does not find all variables references that your Postman collection file is using. Some cases can be: - -- You are using _data_ or _local_ scoped variables, and as stated previously these scopes are not supported by API security testing. Thus, assuming the values for these variables have not been provided through [the API security testing scope](#api-security-testing-scope-custom-json-file-format), then the values of the _data_ and _local_ scoped variables are undefined. -- A variable name was typed incorrectly, and the name does not match the defined variable. -- Postman Client supports a new dynamic variable that is not supported by API security testing. - -When possible, API security testing follows the same behavior as the Postman Client does when dealing with undefined variables. The text of the variable reference remains the same, and there is no text substitution. The same behavior also applies to any unsupported dynamic variables. - -For example, if a request definition in the Postman Collection references the variable `{{full_url}}` and the variable is not found it is left unchanged with the value `{{full_url}}`. - -#### Dynamic Postman variables - -In addition to variables that a user can define at various scope levels, Postman has a set of pre-defined variables called _dynamic_ variables. The [_dynamic_ variables](https://learning.postman.com/docs/writing-scripts/script-references/variables-list/) are already defined and their name is prefixed with a dollar sign (`$`), for instance, `$guid`. _Dynamic_ variables can be used like any other variable, and in the Postman Client, they produce random values during the request/collection run. - -An important difference between API security testing and Postman is that API security testing returns the same value for each usage of the same dynamic variables. This differs from the Postman Client behavior which returns a random value on each use of the same dynamic variable. In other words, API security testing uses static values for dynamic variables while Postman uses random values. - -The supported dynamic variables during the scanning process are: - -| Variable | Value | -| ----------- | ----------- | -| `$guid` | `611c2e81-2ccb-42d8-9ddc-2d0bfa65c1b4` | -| `$isoTimestamp` | `2020-06-09T21:10:36.177Z` | -| `$randomAbbreviation` | `PCI` | -| `$randomAbstractImage` | `http://no-a-valid-host/640/480/abstract` | -| `$randomAdjective` | `auxiliary` | -| `$randomAlphaNumeric` | `a` | -| `$randomAnimalsImage` | `http://no-a-valid-host/640/480/animals` | -| `$randomAvatarImage` | `https://no-a-valid-host/path/to/some/image.jpg` | -| `$randomBankAccount` | `09454073` | -| `$randomBankAccountBic` | `EZIAUGJ1` | -| `$randomBankAccountIban` | `MU20ZPUN3039684000618086155TKZ` | -| `$randomBankAccountName` | `Home Loan Account` | -| `$randomBitcoin` | `3VB8JGT7Y4Z63U68KGGKDXMLLH5` | -| `$randomBoolean` | `true` | -| `$randomBs` | `killer leverage schemas` | -| `$randomBsAdjective` | `viral` | -| `$randomBsBuzz` | `repurpose` | -| `$randomBsNoun` | `markets` | -| `$randomBusinessImage` | `http://no-a-valid-host/640/480/business` | -| `$randomCatchPhrase` | `Future-proofed heuristic open architecture` | -| `$randomCatchPhraseAdjective` | `Business-focused` | -| `$randomCatchPhraseDescriptor` | `bandwidth-monitored` | -| `$randomCatchPhraseNoun` | `superstructure` | -| `$randomCatsImage` | `http://no-a-valid-host/640/480/cats` | -| `$randomCity` | `Spinkahaven` | -| `$randomCityImage` | `http://no-a-valid-host/640/480/city` | -| `$randomColor` | `fuchsia` | -| `$randomCommonFileExt` | `wav` | -| `$randomCommonFileName` | `well_modulated.mpg4` | -| `$randomCommonFileType` | `audio` | -| `$randomCompanyName` | `Grady LLC` | -| `$randomCompanySuffix` | `Inc` | -| `$randomCountry` | `Kazakhstan` | -| `$randomCountryCode` | `MD` | -| `$randomCreditCardMask` | `3622` | -| `$randomCurrencyCode` | `ZMK` | -| `$randomCurrencyName` | `Pound Sterling` | -| `$randomCurrencySymbol` | `£` | -| `$randomDatabaseCollation` | `utf8_general_ci` | -| `$randomDatabaseColumn` | `updatedAt` | -| `$randomDatabaseEngine` | `Memory` | -| `$randomDatabaseType` | `text` | -| `$randomDateFuture` | `Tue Mar 17 2020 13:11:50 GMT+0530 (India Standard Time)` | -| `$randomDatePast` | `Sat Mar 02 2019 09:09:26 GMT+0530 (India Standard Time)` | -| `$randomDateRecent` | `Tue Jul 09 2019 23:12:37 GMT+0530 (India Standard Time)` | -| `$randomDepartment` | `Electronics` | -| `$randomDirectoryPath` | `/usr/local/bin` | -| `$randomDomainName` | `trevor.info` | -| `$randomDomainSuffix` | `org` | -| `$randomDomainWord` | `jaden` | -| `$randomEmail` | `Iva.Kovacek61@no-a-valid-host.com` | -| `$randomExampleEmail` | `non-a-valid-user@example.net` | -| `$randomFashionImage` | `http://no-a-valid-host/640/480/fashion` | -| `$randomFileExt` | `war` | -| `$randomFileName` | `neural_sri_lanka_rupee_gloves.gdoc` | -| `$randomFilePath` | `/home/programming_chicken.cpio` | -| `$randomFileType` | `application` | -| `$randomFirstName` | `Chandler` | -| `$randomFoodImage` | `http://no-a-valid-host/640/480/food` | -| `$randomFullName` | `Connie Runolfsdottir` | -| `$randomHexColor` | `#47594a` | -| `$randomImageDataUri` | `data:image/svg+xml;charset=UTF-8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20version%3D%221.1%22%20baseProfile%3D%22full%22%20width%3D%22undefined%22%20height%3D%22undefined%22%3E%20%3Crect%20width%3D%22100%25%22%20height%3D%22100%25%22%20fill%3D%22grey%22%2F%3E%20%20%3Ctext%20x%3D%220%22%20y%3D%2220%22%20font-size%3D%2220%22%20text-anchor%3D%22start%22%20fill%3D%22white%22%3Eundefinedxundefined%3C%2Ftext%3E%20%3C%2Fsvg%3E` | -| `$randomImageUrl` | `http://no-a-valid-host/640/480` | -| `$randomIngverb` | `navigating` | -| `$randomInt` | `494` | -| `$randomIP` | `241.102.234.100` | -| `$randomIPV6` | `dbe2:7ae6:119b:c161:1560:6dda:3a9b:90a9` | -| `$randomJobArea` | `Mobility` | -| `$randomJobDescriptor` | `Senior` | -| `$randomJobTitle` | `International Creative Liaison` | -| `$randomJobType` | `Supervisor` | -| `$randomLastName` | `Schneider` | -| `$randomLatitude` | `55.2099` | -| `$randomLocale` | `ny` | -| `$randomLongitude` | `40.6609` | -| `$randomLoremLines` | `Ducimus in ut mollitia.\nA itaque non.\nHarum temporibus nihil voluptas.\nIste in sed et nesciunt in quaerat sed.` | -| `$randomLoremParagraph` | `Ab aliquid odio iste quo voluptas voluptatem dignissimos velit. Recusandae facilis qui commodi ea magnam enim nostrum quia quis. Nihil est suscipit assumenda ut voluptatem sed. Esse ab voluptas odit qui molestiae. Rem est nesciunt est quis ipsam expedita consequuntur.` | -| `$randomLoremParagraphs` | `Voluptatem rem magnam aliquam ab id aut quaerat. Placeat provident possimus voluptatibus dicta velit non aut quasi. Mollitia et aliquam expedita sunt dolores nam consequuntur. Nam dolorum delectus ipsam repudiandae et ipsam ut voluptatum totam. Nobis labore labore recusandae ipsam quo.` | -| `$randomLoremSentence` | `Molestias consequuntur nisi non quod.` | -| `$randomLoremSentences` | `Et sint voluptas similique iure amet perspiciatis vero sequi atque. Ut porro sit et hic. Neque aspernatur vitae fugiat ut dolore et veritatis. Ab iusto ex delectus animi. Voluptates nisi iusto. Impedit quod quae voluptate qui.` | -| `$randomLoremSlug` | `eos-aperiam-accusamus, beatae-id-molestiae, qui-est-repellat` | -| `$randomLoremText` | `Quisquam asperiores exercitationem ut ipsum. Aut eius nesciunt. Et reiciendis aut alias eaque. Nihil amet laboriosam pariatur eligendi. Sunt ullam ut sint natus ducimus. Voluptas harum aspernatur soluta rem nam.` | -| `$randomLoremWord` | `est` | -| `$randomLoremWords` | `vel repellat nobis` | -| `$randomMACAddress` | `33:d4:68:5f:b4:c7` | -| `$randomMimeType` | `audio/vnd.vmx.cvsd` | -| `$randomMonth` | `February` | -| `$randomNamePrefix` | `Dr.` | -| `$randomNameSuffix` | `MD` | -| `$randomNatureImage` | `http://no-a-valid-host/640/480/nature` | -| `$randomNightlifeImage` | `http://no-a-valid-host/640/480/nightlife` | -| `$randomNoun` | `bus` | -| `$randomPassword` | `t9iXe7COoDKv8k3` | -| `$randomPeopleImage` | `http://no-a-valid-host/640/480/people` | -| `$randomPhoneNumber` | `700-008-5275` | -| `$randomPhoneNumberExt` | `27-199-983-3864` | -| `$randomPhrase` | `You can't program the monitor without navigating the mobile XML program!` | -| `$randomPrice` | `531.55` | -| `$randomProduct` | `Pizza` | -| `$randomProductAdjective` | `Unbranded` | -| `$randomProductMaterial` | `Steel` | -| `$randomProductName` | `Handmade Concrete Tuna` | -| `$randomProtocol` | `https` | -| `$randomSemver` | `7.0.5` | -| `$randomSportsImage` | `http://no-a-valid-host/640/480/sports` | -| `$randomStreetAddress` | `5742 Harvey Streets` | -| `$randomStreetName` | `Kuhic Island` | -| `$randomTransactionType` | `payment` | -| `$randomTransportImage` | `http://no-a-valid-host/640/480/transport` | -| `$randomUrl` | `https://no-a-valid-host.net` | -| `$randomUserAgent` | `Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.9.8; rv:15.6) Gecko/20100101 Firefox/15.6.6` | -| `$randomUserName` | `Jarrell.Gutkowski` | -| `$randomUUID` | `6929bb52-3ab2-448a-9796-d6480ecad36b` | -| `$randomVerb` | `navigate` | -| `$randomWeekday` | `Thursday` | -| `$randomWord` | `withdrawal` | -| `$randomWords` | `Samoa Synergistic sticky copying Grocery` | -| `$timestamp` | `1562757107` | - -#### Example: Global Scope - -In this example, [the _global_ scope is exported](https://learning.postman.com/docs/sending-requests/variables/#downloading-global-environments) from the Postman Client as `global-scope.json` and provided to API security testing through the `DAST_API_POSTMAN_COLLECTION_VARIABLES` configuration variable. - -Here is an example of using `DAST_API_POSTMAN_COLLECTION_VARIABLES`: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_POSTMAN_COLLECTION: postman-collection.json - DAST_API_POSTMAN_COLLECTION_VARIABLES: global-scope.json - DAST_API_TARGET_URL: http://test-deployment/ -``` - -#### Example: Environment Scope - -In this example, [the _environment_ scope is exported](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-environments) from the Postman Client as `environment-scope.json` and provided to API security testing through the `DAST_API_POSTMAN_COLLECTION_VARIABLES` configuration variable. - -Here is an example of using `DAST_API_POSTMAN_COLLECTION_VARIABLES`: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_POSTMAN_COLLECTION: postman-collection.json - DAST_API_POSTMAN_COLLECTION_VARIABLES: environment-scope.json - DAST_API_TARGET_URL: http://test-deployment/ -``` - -#### Example: Collection Scope - -The _collection_ scope variables are included in the exported Postman Collection file and provided through the `DAST_API_POSTMAN_COLLECTION` configuration variable. - -Here is an example of using `DAST_API_POSTMAN_COLLECTION`: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_POSTMAN_COLLECTION: postman-collection.json - DAST_API_TARGET_URL: http://test-deployment/ -``` - -#### Example: API security testing scope - -The API security testing scope is used for two main purposes, defining _data_ and _local_ scope variables that are not supported by API security testing, and changing the value of an existing variable defined in another scope. The API security testing scope is provided through the `DAST_API_POSTMAN_COLLECTION_VARIABLES` configuration variable. - -Here is an example of using `DAST_API_POSTMAN_COLLECTION_VARIABLES`: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_POSTMAN_COLLECTION: postman-collection.json - DAST_API_POSTMAN_COLLECTION_VARIABLES: dast-api-scope.json - DAST_API_TARGET_URL: http://test-deployment/ -``` - -The file `dast-api-scope.json` uses our [custom JSON file format](#api-security-testing-scope-custom-json-file-format). This JSON is an object with key-value pairs for properties. The keys are the variables' names, and the values are the variables' -values. For example: - -```json -{ - "base_url": "http://127.0.0.1/", - "token": "Token 84816165151" -} -``` - -#### Example: Multiple Scopes - -In this example, a _global_ scope, _environment_ scope, and _collection_ scope are configured. The first step is to export our various scopes. - -- [Export the _global_ scope](https://learning.postman.com/docs/sending-requests/variables/#downloading-global-environments) as `global-scope.json` -- [Export the _environment_ scope](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-environments) as `environment-scope.json` -- Export the Postman Collection which includes the _collection_ scope as `postman-collection.json` - -The Postman Collection is provided using the `DAST_API_POSTMAN_COLLECTION` variable, while the other scopes are provided using the `DAST_API_POSTMAN_COLLECTION_VARIABLES`. API security testing can identify which scope the provided files match using data provided in each file. - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_POSTMAN_COLLECTION: postman-collection.json - DAST_API_POSTMAN_COLLECTION_VARIABLES: global-scope.json,environment-scope.json - DAST_API_TARGET_URL: http://test-deployment/ -``` - -#### Example: Changing a Variables Value - -When using exported scopes, it's often the case that the value of a variable must be changed for use with API security testing. For example, a _collection_ scoped variable might contain a variable named `api_version` with a value of `v2`, while your test needs a value of `v1`. Instead of modifying the exported collection to change the value, the API security testing scope can be used to change its value. This works because the _API security testing_ scope takes precedence over all other scopes. - -The _collection_ scope variables are included in the exported Postman Collection file and provided through the `DAST_API_POSTMAN_COLLECTION` configuration variable. - -The API security testing scope is provided through the `DAST_API_POSTMAN_COLLECTION_VARIABLES` configuration variable, but first, we must create the file. -The file `dast-api-scope.json` uses our [custom JSON file format](#api-security-testing-scope-custom-json-file-format). This JSON is an object with key-value pairs for properties. The keys are the variables' names, and the values are the variables' -values. For example: - -```json -{ - "api_version": "v1" -} -``` - -Our CI definition: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_POSTMAN_COLLECTION: postman-collection.json - DAST_API_POSTMAN_COLLECTION_VARIABLES: dast-api-scope.json - DAST_API_TARGET_URL: http://test-deployment/ -``` - -#### Example: Changing a Variables Value with Multiple Scopes - -When using exported scopes, it's often the case that the value of a variable must be changed for use with API security testing. For example, an _environment_ scope might contain a variable named `api_version` with a value of `v2`, while your test needs a value of `v1`. Instead of modifying the exported file to change the value, the API security testing scope can be used. This works because the _API security testing_ scope takes precedence over all other scopes. - -In this example, a _global_ scope, _environment_ scope, _collection_ scope, and _API security testing_ scope are configured. The first step is to export and create our various scopes. - -- [Export the _global_ scope](https://learning.postman.com/docs/sending-requests/variables/#downloading-global-environments) as `global-scope.json` -- [Export the _environment_ scope](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#exporting-environments) as `environment-scope.json` -- Export the Postman Collection which includes the _collection_ scope as `postman-collection.json` - -The API security testing scope is used by creating a file `dast-api-scope.json` using our [custom JSON file format](#api-security-testing-scope-custom-json-file-format). This JSON is an object with key-value pairs for properties. The keys are the variables' names, and the values are the variables' -values. For example: - -```json -{ - "api_version": "v1" -} -``` - -The Postman Collection is provided using the `DAST_API_POSTMAN_COLLECTION` variable, while the other scopes are provided using the `DAST_API_POSTMAN_COLLECTION_VARIABLES`. API security testing can identify which scope the provided files match using data provided in each file. - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_POSTMAN_COLLECTION: postman-collection.json - DAST_API_POSTMAN_COLLECTION_VARIABLES: global-scope.json,environment-scope.json,dast-api-scope.json - DAST_API_TARGET_URL: http://test-deployment/ -``` - -## Running your first scan - -When configured correctly, a CI/CD pipeline contains a `dast` stage and an `dast_api` job. The job only fails when an invalid configuration is provided. During typical operation, the job always succeeds even if vulnerabilities are identified during testing. - -Vulnerabilities are displayed on the **Security** pipeline tab with the suite name. When testing against the repositories default branch, the API security testing vulnerabilities are also shown on the Security and Compliance's Vulnerability Report page. - -To prevent an excessive number of reported vulnerabilities, the API security testing scanner limits the number of vulnerabilities it reports per operation. - -## Viewing API security testing vulnerabilities - -The API security testing analyzer produces a JSON report that is collected and used -[to populate the vulnerabilities into GitLab vulnerability screens](#view-details-of-an-api-security-testing-vulnerability). - -See [handling false positives](#handling-false-positives) for information about configuration changes you can make to limit the number of false positives reported. - -### View details of an API security testing vulnerability - -Follow these steps to view details of a vulnerability: - -1. You can view vulnerabilities in a project, or a merge request: - - - In a project, go to the project's **Secure > Vulnerability report** - page. This page shows all vulnerabilities from the default branch only. - - In a merge request, go the merge request's **Security** section and select the **Expand** - button. API security testing vulnerabilities are available in a section labeled - **DAST detected N potential vulnerabilities**. Select the title to display the vulnerability - details. - -1. Select the vulnerabilities title to display the details. The table below describes these details. - - | Field | Description | - |:--------------------|:----------------------------------------------------------------------------------------| - | Description | Description of the vulnerability including what was modified. | - | Project | Namespace and project in which the vulnerability was detected. | - | Method | HTTP method used to detect the vulnerability. | - | URL | URL at which the vulnerability was detected. | - | Request | The HTTP request that caused the vulnerability. | - | Unmodified Response | Response from an unmodified request. This is what a typical working response looks like.| - | Actual Response | Response received from test request. | - | Evidence | How we determined a vulnerability occurred. | - | Identifiers | The API security testing check used to find this vulnerability. | - | Severity | Severity of the vulnerability. | - | Scanner Type | Scanner used to perform testing. | - -### Security Dashboard - -The Security Dashboard is a good place to get an overview of all the security vulnerabilities in your groups, projects and -pipelines. For more information, see the [Security Dashboard documentation](../../security_dashboard/index.md). - -### Interacting with the vulnerabilities - -Once a vulnerability is found, you can interact with it. Read more on how to -[address the vulnerabilities](../../vulnerabilities/index.md). - -### Handling False Positives - -False positives can be handled in several ways: - -- Dismiss the vulnerability. -- Some checks have several methods of detecting when a vulnerability is identified, called _Assertions_. - Assertions can also be turned off and configured. For example, the API security testing scanner by default uses HTTP - status codes to help identify when something is a real issue. If an API returns a 500 error during - testing, this creates a vulnerability. This isn't always desired, as some frameworks return 500 errors often. -- Turn off the Check producing the false positive. This prevents the check from generating any - vulnerabilities. Example checks are the SQL Injection Check, and JSON Hijacking Check. - -#### Turn off a Check - -Checks perform testing of a specific type and can be turned on and off for specific configuration -profiles. The provided [configuration files](variables.md#configuration-files) define several profiles that you -can use. The profile definition in the configuration file lists all the checks that are active -during a scan. To turn off a specific check, remove it from the profile definition in the -configuration file. The profiles are defined in the `Profiles` section of the configuration file. - -Example profile definition: - -```yaml -Profiles: - - Name: Quick - DefaultProfile: Empty - Routes: - - Route: *Route0 - Checks: - - Name: ApplicationInformationCheck - - Name: CleartextAuthenticationCheck - - Name: FrameworkDebugModeCheck - - Name: HtmlInjectionCheck - - Name: InsecureHttpMethodsCheck - - Name: JsonHijackingCheck - - Name: JsonInjectionCheck - - Name: SensitiveInformationCheck - - Name: SessionCookieCheck - - Name: SqlInjectionCheck - - Name: TokenCheck - - Name: XmlInjectionCheck -``` - -To turn off the JSON Hijacking Check you can remove these lines: - -```yaml - - Name: JsonHijackingCheck -``` - -This results in the following YAML: - -```yaml -- Name: Quick - DefaultProfile: Empty - Routes: - - Route: *Route0 - Checks: - - Name: ApplicationInformationCheck - - Name: CleartextAuthenticationCheck - - Name: FrameworkDebugModeCheck - - Name: HtmlInjectionCheck - - Name: InsecureHttpMethodsCheck - - Name: JsonInjectionCheck - - Name: SensitiveInformationCheck - - Name: SessionCookieCheck - - Name: SqlInjectionCheck - - Name: TokenCheck - - Name: XmlInjectionCheck -``` - -#### Turn off an Assertion for a Check - -Assertions detect vulnerabilities in tests produced by checks. Many checks support multiple Assertions such as Log Analysis, Response Analysis, and Status Code. When a vulnerability is found, the Assertion used is provided. To identify which Assertions are on by default, see the Checks default configuration in the configuration file. The section is called `Checks`. - -This example shows the SQL Injection Check: - -```yaml -- Name: SqlInjectionCheck - Configuration: - UserInjections: [] - Assertions: - - Name: LogAnalysisAssertion - - Name: ResponseAnalysisAssertion - - Name: StatusCodeAssertion -``` - -Here you can see three Assertions are on by default. A common source of false positives is -`StatusCodeAssertion`. To turn it off, modify its configuration in the `Profiles` section. This -example provides only the other two Assertions (`LogAnalysisAssertion`, -`ResponseAnalysisAssertion`). This prevents `SqlInjectionCheck` from using `StatusCodeAssertion`: - -```yaml -Profiles: - - Name: Quick - DefaultProfile: Empty - Routes: - - Route: *Route0 - Checks: - - Name: ApplicationInformationCheck - - Name: CleartextAuthenticationCheck - - Name: FrameworkDebugModeCheck - - Name: HtmlInjectionCheck - - Name: InsecureHttpMethodsCheck - - Name: JsonHijackingCheck - - Name: JsonInjectionCheck - - Name: SensitiveInformationCheck - - Name: SessionCookieCheck - - Name: SqlInjectionCheck - Assertions: - - Name: LogAnalysisAssertion - - Name: ResponseAnalysisAssertion - - Name: TokenCheck - - Name: XmlInjectionCheck -``` + + + + diff --git a/doc/user/application_security/dast_api/configuration/index.md b/doc/user/application_security/dast_api/configuration/index.md index 189ef14929c..f28350f03d1 100644 --- a/doc/user/application_security/dast_api/configuration/index.md +++ b/doc/user/application_security/dast_api/configuration/index.md @@ -1,15 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments -type: reference, howto +redirect_to: '../../api_security_testing/configuration/index.md' +remove_date: '2024-07-30' --- -# Configuration +This document was moved to [another location](../../api_security_testing/configuration/index.md). -- [Requirements](requirements.md) -- [Enabling the analyzer](enabling_the_analyzer.md) -- [Customize analyzer settings](customizing_analyzer_settings.md) -- [Overriding analyzer jobs](overriding_analyzer_jobs.md) -- [Available CI/CD variables](variables.md) -- [Offline configuration](offline_configuration.md) + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/configuration/offline_configuration.md b/doc/user/application_security/dast_api/configuration/offline_configuration.md index 9bb96bfc384..497e96d4c6a 100644 --- a/doc/user/application_security/dast_api/configuration/offline_configuration.md +++ b/doc/user/application_security/dast_api/configuration/offline_configuration.md @@ -1,30 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/configuration/offline_configuration.md' +remove_date: '2024-07-30' --- -# Offline configuration +This document was moved to [another location](../../api_security_testing/configuration/offline_configuration.md). -For self-managed GitLab instances in an environment with limited, restricted, or intermittent access to external resources through the internet, some adjustments are required for the API security testing job to successfully run. - -Steps: - -1. Host the Docker image in a local container registry. -1. Set the `SECURE_ANALYZERS_PREFIX` to the local container registry. - -The Docker image for API security testing must be pulled (downloaded) from the public registry and then pushed (imported) into a local registry. The GitLab container registry can be used to locally host the Docker image. This process can be performed using a special template. See [loading Docker images onto your offline host](../../offline_deployments/index.md#loading-docker-images-onto-your-offline-host) for instructions. - -Once the Docker image is hosted locally, the `SECURE_ANALYZERS_PREFIX` variable is set with the location of the local registry. The variable must be set such that concatenating `/api-security:2` results in a valid image location. - -NOTE: -API security testing and API Fuzzing both use the same underlying Docker image `api-security:2`. - -For example, the below line sets a registry for the image `registry.gitlab.com/security-products/api-security:2`: - -`SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products"` - -NOTE: -Setting `SECURE_ANALYZERS_PREFIX` changes the Docker image registry location for all GitLab Secure templates. - -For more information, see [Offline environments](../../offline_deployments/index.md). + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/configuration/overriding_analyzer_jobs.md b/doc/user/application_security/dast_api/configuration/overriding_analyzer_jobs.md index 99ca752c923..755fa262ba5 100644 --- a/doc/user/application_security/dast_api/configuration/overriding_analyzer_jobs.md +++ b/doc/user/application_security/dast_api/configuration/overriding_analyzer_jobs.md @@ -1,21 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments -type: reference, howto +redirect_to: '../../api_security_testing/configuration/overriding_analyzer_jobs.md' +remove_date: '2024-07-30' --- -# Overriding API security testing jobs +This document was moved to [another location](../../api_security_testing/configuration/overriding_analyzer_jobs.md). -To override a job definition, (for example, change properties like `variables`, `dependencies`, or [`rules`](../../../../ci/yaml/index.md#rules)), -declare a job with the same name as the DAST job to override. Place this new job after the template -inclusion and specify any additional keys under it. For example, this sets the target APIs base URL: - -```yaml -include: - - template: Security/DAST-API.gitlab-ci.yml - -dast_api: - variables: - DAST_API_TARGET_URL: https://target/api -``` + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/configuration/requirements.md b/doc/user/application_security/dast_api/configuration/requirements.md index a680e8c143c..8096a6e0bcd 100644 --- a/doc/user/application_security/dast_api/configuration/requirements.md +++ b/doc/user/application_security/dast_api/configuration/requirements.md @@ -1,114 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments -type: reference, howto +redirect_to: '../../api_security_testing/configuration/requirements.md' +remove_date: '2024-07-30' --- -# Requirements +This document was moved to [another location](../../api_security_testing/configuration/requirements.md). -- A web API using one of the supported API types: - - REST API - - SOAP - - GraphQL - - Form bodies, JSON, or XML -- An API specification in one of the following formats: - - [OpenAPI v2 or v3 Specification](enabling_the_analyzer.md#openapi-specification) - - [GraphQL Schema](enabling_the_analyzer.md#graphql-schema) - - [HTTP Archive (HAR)](enabling_the_analyzer.md#http-archive-har) - - [Postman Collection v2.0 or v2.1](enabling_the_analyzer.md#postman-collection) -- [GitLab Runner](../../../../ci/runners/index.md) available, with the - [`docker` executor](https://docs.gitlab.com/runner/executors/docker.html) on Linux/amd64. -- Target application deployed. For more details, read [Deployment options](#application-deployment-options). -- `dast` stage added to the CI/CD pipeline definition. This should be added after the deploy step, for example: - - ```yaml - stages: - - build - - test - - deploy - - dast - ``` - -## Recommendations - -- Configure runners to use the [always pull policy](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy) to run the latest versions of the analyzers. -- By default, API security testing downloads all artifacts defined by previous jobs in the pipeline. If - your DAST job does not rely on `environment_url.txt` to define the URL under test or any other files created - in previous jobs, we recommend you don't download artifacts. To avoid downloading - artifacts, extend the analyzer CI/CD job to specify no dependencies. For example, for the DAST proxy-based analyzer add the following to your `.gitlab-ci.yml` file: - - ```yaml - dast_api: - dependencies: [] - ``` - -## Application deployment options - -API security testing requires a deployed application to be available to scan. - -Depending on the complexity of the target application, there are a few options as to how to deploy and configure -the API security testing template. - -### Review Apps - -Review Apps are the most involved method of deploying your DAST target application. To assist in the process, -we created a Review App deployment using Google Kubernetes Engine (GKE). This example can be found in our -[Review Apps - GKE](https://gitlab.com/gitlab-org/security-products/demos/dast/review-app-gke) project, along with detailed -instructions in the [README.md](https://gitlab.com/gitlab-org/security-products/demos/dast/review-app-gke/-/blob/master/README.md) -on how to configure Review Apps for DAST. - -### Docker Services - -If your application uses Docker containers you have another option for deploying and scanning with DAST. -After your Docker build job completes and your image is added to your container registry, you can use the image as a -[service](../../../../ci/services/index.md). - -By using service definitions in your `.gitlab-ci.yml`, you can scan services with the DAST analyzer. - -When adding a `services` section to the job, the `alias` is used to define the hostname that can be used to access the service. In the following example, the `alias: yourapp` portion of the `dast` job definition means that the URL to the deployed application uses `yourapp` as the hostname (`https://yourapp/`). - -```yaml -stages: - - build - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -# Deploys the container to the GitLab container registry -deploy: - services: - - name: docker:dind - alias: dind - image: docker:20.10.16 - stage: build - script: - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - - docker pull $CI_REGISTRY_IMAGE:latest || true - - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA --tag $CI_REGISTRY_IMAGE:latest . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE:latest - -dast_api: - services: # use services to link your app container to the dast job - - name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - alias: yourapp - -variables: - DAST_API_TARGET_URL: https://yourapp -``` - -Most applications depend on multiple services such as databases or caching services. By default, services defined in the services fields cannot communicate -with each another. To allow communication between services, enable the `FF_NETWORK_PER_BUILD` [feature flag](https://docs.gitlab.com/runner/configuration/feature-flags.html#available-feature-flags). - -```yaml -variables: - FF_NETWORK_PER_BUILD: "true" # enable network per build so all services can communicate on the same network - -services: # use services to link the container to the dast job - - name: mongo:latest - alias: mongo - - name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - alias: yourapp -``` + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/configuration/variables.md b/doc/user/application_security/dast_api/configuration/variables.md index ceaaca8c75e..abde4312722 100644 --- a/doc/user/application_security/dast_api/configuration/variables.md +++ b/doc/user/application_security/dast_api/configuration/variables.md @@ -1,104 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../api_security_testing/configuration/variables.md' +remove_date: '2024-07-30' --- -# Available CI/CD variables and configuration files +This document was moved to [another location](../../api_security_testing/configuration/variables.md). -## Available CI/CD variables - -| CI/CD variable | Description | -|------------------------------------------------------|--------------------| -| `SECURE_ANALYZERS_PREFIX` | Specify the Docker registry base address from which to download the analyzer. | -| `DAST_API_DISABLED` | Set to 'true' or '1' to disable API security testing scanning. | -| `DAST_API_DISABLED_FOR_DEFAULT_BRANCH` | Set to 'true' or '1' to disable API security testing scanning for only the default (production) branch. | -| `DAST_API_VERSION` | Specify API security testing container version. Defaults to `3`. | -| `DAST_API_IMAGE_SUFFIX` | Specify a container image suffix. Defaults to none. | -| `DAST_API_API_PORT` | Specify the communication port number used by API security testing engine. Defaults to `5500`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367734) in GitLab 15.5. | -| `DAST_API_TARGET_URL` | Base URL of API testing target. | -|[`DAST_API_CONFIG`](#configuration-files) | API security testing configuration file. Defaults to `.gitlab-dast-api.yml`. | -|[`DAST_API_PROFILE`](#configuration-files) | Configuration profile to use during testing. Defaults to `Quick`. | -|[`DAST_API_EXCLUDE_PATHS`](customizing_analyzer_settings.md#exclude-paths) | Exclude API URL paths from testing. | -|[`DAST_API_EXCLUDE_URLS`](customizing_analyzer_settings.md#exclude-urls) | Exclude API URL from testing. | -|[`DAST_API_EXCLUDE_PARAMETER_ENV`](customizing_analyzer_settings.md#exclude-parameters) | JSON string containing excluded parameters. | -|[`DAST_API_EXCLUDE_PARAMETER_FILE`](customizing_analyzer_settings.md#exclude-parameters) | Path to a JSON file containing excluded parameters. | -|[`DAST_API_REQUEST_HEADERS`](customizing_analyzer_settings.md#request-headers) | A comma-separated (`,`) list of headers to include on each scan request. Consider using `DAST_API_REQUEST_HEADERS_BASE64` when storing secret header values in a [masked variable](../../../../ci/variables/index.md#mask-a-cicd-variable), which has character set restrictions. | -|[`DAST_API_REQUEST_HEADERS_BASE64`](customizing_analyzer_settings.md#request-headers) | A comma-separated (`,`) list of headers to include on each scan request, Base64-encoded. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/378440) in GitLab 15.6. | -|[`DAST_API_OPENAPI`](enabling_the_analyzer.md#openapi-specification) | OpenAPI specification file or URL. | -|[`DAST_API_OPENAPI_RELAXED_VALIDATION`](enabling_the_analyzer.md#openapi-specification) | Relax document validation. Default is disabled. | -|[`DAST_API_OPENAPI_ALL_MEDIA_TYPES`](enabling_the_analyzer.md#openapi-specification) | Use all supported media types instead of one when generating requests. Causes test duration to be longer. Default is disabled. | -|[`DAST_API_OPENAPI_MEDIA_TYPES`](enabling_the_analyzer.md#openapi-specification) | Colon (`:`) separated media types accepted for testing. Default is disabled. | -|[`DAST_API_HAR`](enabling_the_analyzer.md#http-archive-har) | HTTP Archive (HAR) file. | -|[`DAST_API_GRAPHQL`](enabling_the_analyzer.md#graphql-schema) | Path to GraphQL endpoint, for example `/api/graphql`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352780) in GitLab 15.4. | -|[`DAST_API_GRAPHQL_SCHEMA`](enabling_the_analyzer.md#graphql-schema) | A URL or filename for a GraphQL schema in JSON format. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352780) in GitLab 15.4. | -|[`DAST_API_POSTMAN_COLLECTION`](enabling_the_analyzer.md#postman-collection) | Postman Collection file. | -|[`DAST_API_POSTMAN_COLLECTION_VARIABLES`](enabling_the_analyzer.md#postman-variables) | Path to a JSON file to extract Postman variable values. The support for comma-separated (`,`) files was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356312) in GitLab 15.1. | -|[`DAST_API_OVERRIDES_FILE`](customizing_analyzer_settings.md#overrides) | Path to a JSON file containing overrides. | -|[`DAST_API_OVERRIDES_ENV`](customizing_analyzer_settings.md#overrides) | JSON string containing headers to override. | -|[`DAST_API_OVERRIDES_CMD`](customizing_analyzer_settings.md#overrides) | Overrides command. | -|[`DAST_API_OVERRIDES_CMD_VERBOSE`](customizing_analyzer_settings.md#overrides) | When set to any value. It shows overrides command output as part of the job output. | -|`DAST_API_PRE_SCRIPT` | Run user command or script before scan session starts. | -|`DAST_API_POST_SCRIPT` | Run user command or script after scan session has finished. | -|[`DAST_API_OVERRIDES_INTERVAL`](customizing_analyzer_settings.md#overrides) | How often to run overrides command in seconds. Defaults to `0` (once). | -|[`DAST_API_HTTP_USERNAME`](customizing_analyzer_settings.md#http-basic-authentication) | Username for HTTP authentication. | -|[`DAST_API_HTTP_PASSWORD`](customizing_analyzer_settings.md#http-basic-authentication) | Password for HTTP authentication. Consider using `DAST_API_HTTP_PASSWORD_BASE64` instead. | -|[`DAST_API_HTTP_PASSWORD_BASE64`](customizing_analyzer_settings.md#http-basic-authentication) | Password for HTTP authentication, base64-encoded. [Introduced](https://gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing-src/-/merge_requests/702) in GitLab 15.4. | -|`DAST_API_SERVICE_START_TIMEOUT` | How long to wait for target API to become available in seconds. Default is 300 seconds. | -|`DAST_API_TIMEOUT` | How long to wait for API responses in seconds. Default is 30 seconds. | - -## Configuration files - -To get you started quickly, GitLab provides the configuration file -[`gitlab-dast-api-config.yml`](https://gitlab.com/gitlab-org/security-products/analyzers/dast/-/blob/master/config/gitlab-dast-api-config.yml). -This file has several testing profiles that perform various numbers of tests. The run time of each -profile increases as the test numbers go up. To use a configuration file, add it to your -repository's root as `.gitlab/gitlab-dast-api-config.yml`. - -### Profiles - -The following profiles are pre-defined in the default configuration file. Profiles -can be added, removed, and modified by creating a custom configuration. - -#### Passive - -- Application Information Check -- Cleartext Authentication Check -- JSON Hijacking Check -- Sensitive Information Check -- Session Cookie Check - -#### Quick - -- Application Information Check -- Cleartext Authentication Check -- FrameworkDebugModeCheck -- HTML Injection Check -- Insecure Http Methods Check -- JSON Hijacking Check -- JSON Injection Check -- Sensitive Information Check -- Session Cookie Check -- SQL Injection Check -- Token Check -- XML Injection Check - -#### Full - -- Application Information Check -- Cleartext AuthenticationCheck -- CORS Check -- DNS Rebinding Check -- Framework Debug Mode Check -- HTML Injection Check -- Insecure Http Methods Check -- JSON Hijacking Check -- JSON Injection Check -- Open Redirect Check -- Sensitive File Check -- Sensitive Information Check -- Session Cookie Check -- SQL Injection Check -- TLS Configuration Check -- Token Check -- XML Injection Check + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/index.md b/doc/user/application_security/dast_api/index.md index e51b6ddfc90..b152a65374f 100644 --- a/doc/user/application_security/dast_api/index.md +++ b/doc/user/application_security/dast_api/index.md @@ -1,85 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../api_security_testing/index.md' +remove_date: '2024-07-30' --- -# API security testing analyzer +This document was moved to [another location](../api_security_testing/index.md). -DETAILS: -**Tier:** Ultimate -**Offering:** GitLab.com, Self-managed, GitLab Dedicated - -> - API security testing analyzer [became the default analyzer for on-demand API security testing scans](https://gitlab.com/groups/gitlab-org/-/epics/4254) in GitLab 15.6. -> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/457449) from **DAST API analyzer** to **API security testing analyzer** in GitLab 17.0. - -Perform Dynamic Application Security Testing (DAST) of web APIs to help discover bugs and potential -security issues that other QA processes may miss. Use API security testing in addition to -other [GitLab Secure](../index.md) security scanners and your own test processes. You can run DAST -API tests either as part your CI/CD workflow, [on-demand](../dast/on-demand_scan.md), or both. - -WARNING: -Do not run API security testing against a production server. Not only can it perform _any_ function that -the API can, it may also trigger bugs in the API. This includes actions like modifying and deleting -data. Only run API security testing against a test server. - -API security testing can test the following web API types: - -- REST API -- SOAP -- GraphQL -- Form bodies, JSON, or XML - -## When API security testing scans run - -When run in your CI/CD pipeline, API security testing scanning runs in the `dast` stage by default. To ensure -API security testing scanning examines the latest code, ensure your CI/CD pipeline deploys changes to a test -environment in a stage before the `dast` stage. - -If your pipeline is configured to deploy to the same web server on each run, running a pipeline -while another is still running could cause a race condition in which one pipeline overwrites the -code from another. The API to be scanned should be excluded from changes for the duration of a -API security testing scan. The only changes to the API should be from the API security testing scanner. Changes made to the -API (for example, by users, scheduled tasks, database changes, code changes, other pipelines, or -other scanners) during a scan could cause inaccurate results. - -## Example API security testing scanning configurations - -The following projects demonstrate API security testing scanning: - -- [Example OpenAPI v2 Specification project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/openapi-example) -- [Example HTTP Archive (HAR) project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/har-example) -- [Example Postman Collection project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/postman-example) -- [Example GraphQL project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/graphql-example) -- [Example SOAP project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/soap-example) -- [Authentication Token using Selenium](https://gitlab.com/gitlab-org/security-products/demos/api-dast/auth-token-selenium) - -## Get support or request an improvement - -To get support for your particular problem, use the [getting help channels](https://about.gitlab.com/get-help/). - -The [GitLab issue tracker on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues) is the right place for bugs and feature proposals about API Security and API security testing. -Use `~"Category:API Security"` [label](../../../development/labels/index.md) when opening a new issue regarding API security testing to ensure it is quickly reviewed by the right people. Refer to our [review response SLO](https://handbook.gitlab.com/handbook/engineering/workflow/code-review/#review-response-slo) to understand when you should receive a response. - -[Search the issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues) for similar entries before submitting your own, there's a good chance somebody else had the same issue or feature proposal. Show your support with an emoji reaction or join the discussion. - -When experiencing a behavior not working as expected, consider providing contextual information: - -- GitLab version if using a self-managed instance. -- `.gitlab-ci.yml` job definition. -- Full job console output. -- Scanner log file available as a job artifact named `gl-api-security-scanner.log`. - -WARNING: -**Sanitize data attached to a support issue**. Remove sensitive information, including: credentials, passwords, tokens, keys, and secrets. - -## Glossary - -- Assert: Assertions are detection modules used by checks to trigger a vulnerability. Many assertions have - configurations. A check can use multiple Assertions. For example, Log Analysis, Response Analysis, - and Status Code are common Assertions used together by checks. Checks with multiple Assertions - allow them to be turned on and off. -- Check: Performs a specific type of test, or performed a check for a type of vulnerability. For - example, the SQL Injection Check performs DAST testing for SQL Injection vulnerabilities. The API security testing scanner is comprised of several checks. Checks can be turned on and off in a profile. -- Profile: A configuration file has one or more testing profiles, or sub-configurations. You may - have a profile for feature branches and another with extra testing for a main branch. + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/performance.md b/doc/user/application_security/dast_api/performance.md index 8c3792dd3ec..a2763938022 100644 --- a/doc/user/application_security/dast_api/performance.md +++ b/doc/user/application_security/dast_api/performance.md @@ -1,219 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../api_security_testing/performance.md' +remove_date: '2024-07-30' --- -# Performance tuning and testing speed +This document was moved to [another location](../api_security_testing/performance.md). -Security tools that perform dynamic analysis testing, such as API security testing, perform testing by sending requests to an instance of your running application. The requests are engineered to test for specific vulnerabilities that might exist in your application. The speed of a dynamic analysis test depends on the following: - -- How many requests per second can be sent to your application by our tooling -- How fast your application responds to requests -- How many requests must be sent to test the application - - How many operations your API is comprised of - - How many fields are in each operation (think JSON bodies, headers, query string, cookies, etc.) - -If the API security testing job still takes longer than expected reach after following the advice in this performance guide, reach out to support for further assistance. - -## Diagnosing performance issues - -The first step to resolving performance issues is to understand what is contributing to the slower-than-expected testing time. Some common issues we see are: - -- API security testing is running on a low-vCPU runner -- The application deployed to a slow/single-CPU instance and is not able to keep up with the testing load -- The application contains a slow operation that impacts the overall test speed (> 1/2 second) -- The application contains an operation that returns a large amount of data (> 500K+) -- The application contains a large number of operations (> 40) - -### The application contains a slow operation that impacts the overall test speed (> 1/2 second) - -The API security testing job output contains helpful information about how fast we are testing, how fast each operation being tested responds, and summary information. Let's take a look at some sample output to see how it can be used in tracking down performance issues: - -```shell -DAST API: Loaded 10 operations from: assets/har-large-response/large_responses.har -DAST API: -DAST API: Testing operation [1/10]: 'GET http://target:7777/api/large_response_json'. -DAST API: - Parameters: (Headers: 4, Query: 0, Body: 0) -DAST API: - Request body size: 0 Bytes (0 bytes) -DAST API: -DAST API: Finished testing operation 'GET http://target:7777/api/large_response_json'. -DAST API: - Excluded Parameters: (Headers: 0, Query: 0, Body: 0) -DAST API: - Performed 767 requests -DAST API: - Average response body size: 130 MB -DAST API: - Average call time: 2 seconds and 82.69 milliseconds (2.082693 seconds) -DAST API: - Time to complete: 14 minutes, 8 seconds and 788.36 milliseconds (848.788358 seconds) -``` - -This job console output snippet starts by telling us how many operations were found (10), followed by notifications that testing has started on a specific operation and a summary of the operation has been completed. The summary is the most interesting part of this log output. In the summary, we can see that it took API security testing 767 requests to fully test this operation and its related fields. We can also see that the average response time was 2 seconds and the time to complete was 14 minutes for this one operation. - -An average response time of 2 seconds is a good initial indicator that this specific operation takes a long time to test. Further, we can see that the response body size is quite large. The large body size is the culprit here, transferring that much data on each request is what takes the majority of that 2 seconds. - -For this issue, the team might decide to: - -- Use a runner with more vCPUs, as this allows API security testing to parallelize the work being performed. This helps lower the test time, but getting the test down under 10 minutes might still be problematic without moving to a high CPU machine due to how long the operation takes to test. While larger runners are more costly, you also pay for less minutes if the job executions are quicker. -- [Exclude this operation](#excluding-slow-operations) from API security testing. While this is the simplest, it has the downside of a gap in security test coverage. -- [Exclude the operation from feature branch API security testing, but include it in the default branch test](#excluding-operations-in-feature-branches-but-not-default-branch). -- [Split up API security testing into multiple jobs](#splitting-a-test-into-multiple-jobs). - -The likely solution is to use a combination of these solutions to reach an acceptable test time, assuming your team's requirements are in the 5-7 minute range. - -## Addressing performance issues - -The following sections document various options for addressing performance issues for API security testing: - -- [Using a larger runner](#using-a-larger-runner) -- [Excluding slow operations](#excluding-slow-operations) -- [Splitting a test into multiple jobs](#splitting-a-test-into-multiple-jobs) -- [Excluding operations in feature branches, but not default branch](#excluding-operations-in-feature-branches-but-not-default-branch) - -### Using a larger runner - -One of the easiest performance boosts can be achieved using a [larger runner](../../../ci/runners/hosted_runners/linux.md#machine-types-available-for-linux-x86-64) with API security testing. This table shows statistics collected during benchmarking of a Java Spring Boot REST API. In this benchmark, the target and API security testing share a single runner instance. - -| Hosted runner on Linux tag | Requests per Second | -|------------------------------------|-----------| -| `saas-linux-small-amd64` (default) | 255 | -| `saas-linux-medium-amd64` | 400 | - -As we can see from this table, increasing the size of the runner and vCPU count can have a large impact on testing speed/performance. - -Here is an example job definition for API security testing that adds a `tags` section to use the medium SaaS runner on Linux. The job extends the job definition included through the API security testing template. - -```yaml -dast_api: - tags: - - saas-linux-medium-amd64 -``` - -In the `gl-api-security-scanner.log` file you can search for the string `Starting work item processor` to inspect the reported max DOP (degree of parallelism). The max DOP should be greater than or equal to the number of vCPUs assigned to the runner. If unable to identify the problem, open a ticket with support to assist. - -Example log entry: - -`17:00:01.084 [INF] Starting work item processor with 4 max DOP` - -### Excluding slow operations - -In the case of one or two slow operations, the team might decide to skip testing the operations. Excluding the operation is done using the `DAST_API_EXCLUDE_PATHS` configuration [variable as explained in this section.](configuration/customizing_analyzer_settings.md#exclude-paths) - -In this example, we have an operation that returns a large amount of data. The operation is `GET http://target:7777/api/large_response_json`. To exclude it we provide the `DAST_API_EXCLUDE_PATHS` configuration variable with the path portion of our operation URL `/api/large_response_json`. - -To verify the operation is excluded, run the API security testing job and review the job console output. It includes a list of included and excluded operations at the end of the test. - -```yaml -dast_api: - variables: - DAST_API_EXCLUDE_PATHS: /api/large_response_json -``` - -Excluding operations from testing could allow some vulnerabilities to go undetected. -{: .alert .alert-warning} - -### Splitting a test into multiple jobs - -Splitting a test into multiple jobs is supported by API security testing through the use of [`DAST_API_EXCLUDE_PATHS`](configuration/customizing_analyzer_settings.md#exclude-paths) and [`DAST_API_EXCLUDE_URLS`](configuration/customizing_analyzer_settings.md#exclude-urls). When splitting a test up, a good pattern is to disable the `dast_api` job and replace it with two jobs with identifying names. In this example we have two jobs, each job is testing a version of the API, so our names reflect that. However, this technique can be applied to any situation, not just with versions of an API. - -The rules we are using in the `dast_api_v1` and `dast_api_v2` jobs are copied from the [API security testing template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml). - -```yaml -# Disable the main dast_api job -dast_api: - rules: - - if: $CI_COMMIT_BRANCH - when: never - -dast_api_v1: - extends: dast_api - variables: - DAST_API_EXCLUDE_PATHS: /api/v1/** - rules: - - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1' - when: never - - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $CI_COMMIT_BRANCH && - $CI_GITLAB_FIPS_MODE == "true" - variables: - DAST_API_IMAGE_SUFFIX: "-fips" - - if: $CI_COMMIT_BRANCH - -dast_api_v2: - variables: - DAST_API_EXCLUDE_PATHS: /api/v2/** - rules: - - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1' - when: never - - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $CI_COMMIT_BRANCH && - $CI_GITLAB_FIPS_MODE == "true" - variables: - DAST_API_IMAGE_SUFFIX: "-fips" - - if: $CI_COMMIT_BRANCH -``` - -### Excluding operations in feature branches, but not default branch - -In the case of one or two slow operations, the team might decide to skip testing the operations, or exclude them from feature branch tests, but include them for default branch tests. Excluding the operation is done using the `DAST_API_EXCLUDE_PATHS` configuration [variable as explained in this section.](configuration/customizing_analyzer_settings.md#exclude-paths) - -In this example, we have an operation that returns a large amount of data. The operation is `GET http://target:7777/api/large_response_json`. To exclude it we provide the `DAST_API_EXCLUDE_PATHS` configuration variable with the path portion of our operation URL `/api/large_response_json`. Our configuration disables the main `dast_api` job and creates two new jobs `dast_api_main` and `dast_api_branch`. The `dast_api_branch` is set up to exclude the long operation and only run on non-default branches (for example, feature branches). The `dast_api_main` branch is set up to only execute on the default branch (`main` in this example). The `dast_api_branch` jobs run faster, allowing for quick development cycles, while the `dast_api_main` job which only runs on default branch builds, takes longer to run. - -To verify the operation is excluded, run the API security testing job and review the job console output. It includes a list of included and excluded operations at the end of the test. - -```yaml -# Disable the main job so we can create two jobs with -# different names -dast_api: - rules: - - if: $CI_COMMIT_BRANCH - when: never - -# API security testing for feature branch work, excludes /api/large_response_json -dast_api_branch: - extends: dast_api - variables: - DAST_API_EXCLUDE_PATHS: /api/large_response_json - rules: - - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1' - when: never - - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $CI_COMMIT_BRANCH && - $CI_GITLAB_FIPS_MODE == "true" - variables: - DAST_API_IMAGE_SUFFIX: "-fips" - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - when: never - - if: $CI_COMMIT_BRANCH - -# API security testing for default branch (main in our case) -# Includes the long running operations -dast_api_main: - extends: dast_api - rules: - - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1' - when: never - - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $CI_COMMIT_BRANCH && - $CI_GITLAB_FIPS_MODE == "true" - variables: - DAST_API_IMAGE_SUFFIX: "-fips" - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH -``` + + + + \ No newline at end of file diff --git a/doc/user/application_security/dast_api/troubleshooting.md b/doc/user/application_security/dast_api/troubleshooting.md index 78bfb1feeed..d85a4344038 100644 --- a/doc/user/application_security/dast_api/troubleshooting.md +++ b/doc/user/application_security/dast_api/troubleshooting.md @@ -1,302 +1,11 @@ --- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../api_security_testing/troubleshooting.md' +remove_date: '2024-07-30' --- -# Troubleshooting +This document was moved to [another location](../api_security_testing/troubleshooting.md). -## API security testing job times out after N hours - -For larger repositories, the API security testing job could time out on the [small hosted runner on Linux](../../../ci/runners/hosted_runners/linux.md#machine-types-available-for-linux-x86-64), which is set per default. If this happens in your jobs, you should scale up to a [larger runner](performance.md#using-a-larger-runner). - -See the following documentation sections for assistance: - -- [Performance tuning and testing speed](performance.md#performance-tuning-and-testing-speed) -- [Using a larger Runner](performance.md#using-a-larger-runner) -- [Excluding operations by path](configuration/customizing_analyzer_settings.md#exclude-paths) -- [Excluding slow operations](performance.md#excluding-slow-operations) - -## API security testing job takes too long to complete - -See [Performance Tuning and Testing Speed](performance.md#performance-tuning-and-testing-speed) - -## Error: `Error waiting for DAST API 'http://127.0.0.1:5000' to become available` - -A bug exists in versions of the API security testing analyzer prior to v1.6.196 that can cause a background process to fail under certain conditions. The solution is to update to a newer version of the API security testing analyzer. - -The version information can be found in the job details for the `dast_api` job. - -If the issue is occurring with versions v1.6.196 or greater, contact Support and provide the following information: - -1. Reference this troubleshooting section and ask for the issue to be escalated to the Dynamic Analysis Team. -1. The full console output of the job. -1. The `gl-api-security-scanner.log` file available as a job artifact. In the right-hand panel of the job details page, select the **Browse** button. -1. The `dast_api` job definition from your `.gitlab-ci.yml` file. - -**Error message** - -- In [GitLab 15.6 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/376078), `Error waiting for DAST API 'http://127.0.0.1:5000' to become available` -- In GitLab 15.5 and earlier, `Error waiting for API Security 'http://127.0.0.1:5000' to become available`. - -## `Failed to start scanner session (version header not found)` - -The API security testing engine outputs an error message when it cannot establish a connection with the scanner application component. The error message is shown in the job output window of the `dast_api` job. A common cause of this issue is changing the `DAST_API_API` variable from its default. - -**Error message** - -- `Failed to start scanner session (version header not found).` - -**Solution** - -- Remove the `DAST_API_API` variable from the `.gitlab-ci.yml` file. The value inherits from the API security testing CI/CD template. We recommend this method instead of manually setting a value. -- If removing the variable is not possible, check to see if this value has changed in the latest version of the [API security testing CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml). If so, update the value in the `.gitlab-ci.yml` file. - -## `Failed to start session with scanner. Please retry, and if the problem persists reach out to support.` - -The API security testing engine outputs an error message when it cannot establish a connection with the scanner application component. The error message is shown in the job output window of the `dast_api` job. A common cause for this issue is that the background component cannot use the selected port as it's already in use. This error can occur intermittently if timing plays a part (race condition). This issue occurs most often with Kubernetes environments when other services are mapped into the container causing port conflicts. - -Before proceeding with a solution, it is important to confirm that the error message was produced because the port was already taken. To confirm this was the cause: - -1. Go to the job console. - -1. Look for the artifact `gl-api-security-scanner.log`. You can either download all artifacts by selecting **Download** and then search for the file, or directly start searching by selecting **Browse**. - -1. Open the file `gl-api-security-scanner.log` in a text editor. - -1. If the error message was produced because the port was already taken, you should see in the file a message like the following: - -- In [GitLab 15.5 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/367734): - - ```log - Failed to bind to address http://127.0.0.1:5500: address already in use. - ``` - -- In GitLab 15.4 and earlier: - - ```log - Failed to bind to address http://[::]:5000: address already in use. - ``` - -The text `http://[::]:5000` in the previous message could be different in your case, for instance it could be `http://[::]:5500` or `http://127.0.0.1:5500`. As long as the remaining parts of the error message are the same, it is safe to assume the port was already taken. - -If you did not find evidence that the port was already taken, check other troubleshooting sections which also address the same error message shown in the job console output. If there are no more options, feel free to [get support or request an improvement](index.md#get-support-or-request-an-improvement) through the proper channels. - -Once you have confirmed the issue was produced because the port was already taken. Then, [GitLab 15.5 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/367734) introduced the configuration variable `DAST_API_API_PORT`. This configuration variable allows setting a fixed port number for the scanner background component. - -**Solution** - -1. Ensure your `.gitlab-ci.yml` file defines the configuration variable `DAST_API_API_PORT`. -1. Update the value of `DAST_API_API_PORT` to any available port number greater than 1024. We recommend checking that the new value is not in used by GitLab. See the full list of ports used by GitLab in [Package defaults](../../../administration/package_information/defaults.md#ports) - -## `Application cannot determine the base URL for the target API` - -The API security testing engine outputs an error message when it cannot determine the target API after inspecting the OpenAPI document. This error message is shown when the target API has not been set in the `.gitlab-ci.yml` file, it is not available in the `environment_url.txt` file, and it could not be computed using the OpenAPI document. - -There is a order of precedence in which the API security testing engine tries to get the target API when checking the different sources. First, it tries to use the `DAST_API_TARGET_URL`. If the environment variable has not been set, then the API security testing engine attempts to use the `environment_url.txt` file. If there is no file `environment_url.txt`, then the API security testing engine uses the OpenAPI document contents and the URL provided in `DAST_API_OPENAPI` (if a URL is provided) to try to compute the target API. - -The best-suited solution depends on whether or not your target API changes for each deployment. In static environments, the target API is the same for each deployment, in this case refer to the [static environment solution](#static-environment-solution). If the target API changes for each deployment a [dynamic environment solution](#dynamic-environment-solutions) should be applied. - -## API security testing job excludes some paths from operations - -If you find that some paths are being excluded from operations, ensure that the `consumes` array is defined and has a valid type in the target definition JSON file. This is required. - -See the [example project target definition file](https://gitlab.com/gitlab-org/security-products/demos/api-dast/openapi-example/-/blob/12e2b039d08208f1dd38a1e7c52b0bda848bb449/rest_target_openapi.json?plain=1#L13) where the `consumes` array is defined. - -### Static environment solution - -This solution is for pipelines in which the target API URL doesn't change (is static). - -**Add environmental variable** - -For environments where the target API remains the same, we recommend you specify the target URL by using the `DAST_API_TARGET_URL` environment variable. In your `.gitlab-ci.yml`, add a variable `DAST_API_TARGET_URL`. The variable must be set to the base URL of API testing target. For example: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_OPENAPI: test-api-specification.json -``` - -### Dynamic environment solutions - -In a dynamic environment your target API changes for each different deployment. In this case, there is more than one possible solution, we recommend you use the `environment_url.txt` file when dealing with dynamic environments. - -**Use environment_url.txt** - -To support dynamic environments in which the target API URL changes during each pipeline, API security testing engine supports the use of an `environment_url.txt` file that contains the URL to use. This file is not checked into the repository, instead it's created during the pipeline by the job that deploys the test target and collected as an artifact that can be used by later jobs in the pipeline. The job that creates the `environment_url.txt` file must run before the API security testing engine job. - -1. Modify the test target deployment job adding the base URL in an `environment_url.txt` file at the root of your project. -1. Modify the test target deployment job collecting the `environment_url.txt` as an artifact. - -Example: - -```yaml -deploy-test-target: - script: - # Perform deployment steps - # Create environment_url.txt (example) - - echo http://${CI_PROJECT_ID}-${CI_ENVIRONMENT_SLUG}.example.org > environment_url.txt - - artifacts: - paths: - - environment_url.txt -``` - -## Use OpenAPI with an invalid schema - -There are cases where the document is autogenerated with an invalid schema or cannot be edited manually in a timely manner. In those scenarios, the API security testing is able to perform a relaxed validation by setting the variable `DAST_API_OPENAPI_RELAXED_VALIDATION`. We recommend providing a fully compliant OpenAPI document to prevent unexpected behaviors. - -### Edit a non-compliant OpenAPI file - -To detect and correct elements that don't comply with the OpenAPI specifications, we recommend using an editor. An editor commonly provides document validation, and suggestions to create a schema-compliant OpenAPI document. Suggested editors include: - -| Editor | OpenAPI 2.0 | OpenAPI 3.0.x | OpenAPI 3.1.x | -| -- | -- | -- | -- | -| [Swagger Editor](https://editor.swagger.io/) | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | **{dotted-circle}** YAML, JSON | -| [Stoplight Studio](https://stoplight.io/studio) | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | - -If your OpenAPI document is generated manually, load your document in the editor and fix anything that is non-compliant. If your document is generated automatically, load it in your editor to identify the issues in the schema, then go to the application and perform the corrections based on the framework you are using. - -### Enable OpenAPI relaxed validation - -Relaxed validation is meant for cases when the OpenAPI document cannot meet OpenAPI specifications, but it still has enough content to be consumed by different tools. A validation is performed but less strictly in regards to document schema. - -API security testing can still try to consume an OpenAPI document that does not fully comply with OpenAPI specifications. To instruct API security testing to perform a relaxed validation, set the variable `DAST_API_OPENAPI_RELAXED_VALIDATION` to any value, for example: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_PROFILE: Quick - DAST_API_TARGET_URL: http://test-deployment/ - DAST_API_OPENAPI: test-api-specification.json - DAST_API_OPENAPI_RELAXED_VALIDATION: 'On' -``` - -## `No operation in the OpenAPI document is consuming any supported media type` - -API security testing uses the specified media types in the OpenAPI document to generate requests. If no request can be created due to the lack of supported media types, then an error is thrown. - -**Error message** - -- `Error, no operation in the OpenApi document is consuming any supported media type. Check 'OpenAPI Specification' to check the supported media types.` - -**Solution** - -1. Review supported media types in the [OpenAPI Specification](configuration/enabling_the_analyzer.md#openapi-specification) section. -1. Edit your OpenAPI document, allowing at least a given operation to accept any of the supported media types. Alternatively, a supported media type could be set in the OpenAPI document level and get applied to all operations. This step may require changes in your application to ensure the supported media type is accepted by the application. - -## ``Error, error occurred trying to download ``: There was an error when retrieving content from Uri:' '. Error:The SSL connection could not be established, see inner exception.`` - -API security testing is compatible with a broad range of TLS configurations, including outdated protocols and ciphers. -Despite broad support, you might encounter connection errors. This error occurs because API security testing could not establish a secure connection with the server at the given URL. - -To resolve the issue: - -If the host in the error message supports non-TLS connections, change `https://` to `http://` in your configuration. -For example, if an error occurs with the following configuration: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_TARGET_URL: https://test-deployment/ - DAST_API_OPENAPI: https://specs/openapi.json -``` - -Change the prefix of `DAST_API_OPENAPI` from `https://` to `http://`: - -```yaml -stages: - - dast - -include: - - template: DAST-API.gitlab-ci.yml - -variables: - DAST_API_TARGET_URL: https://test-deployment/ - DAST_API_OPENAPI: http://specs/openapi.json -``` - -If you cannot use a non-TLS connection to access the URL, contact the Support team for help. - -You can expedite the investigation with the [testssl.sh tool](https://testssl.sh/). From a machine with a bash shell and connectivity to the affected server: - -1. Download the latest release `zip` or `tar.gz` file and extract from . -1. Run `./testssl.sh --log https://specs`. -1. Attach the log file to your support ticket. - -## `ERROR: Job failed: failed to pull image` - -This error message occurs when pulling an image from a container registry that requires authentication to access (it is not public). - -In the job console output the error looks like: - -```log -Running with gitlab-runner 15.6.0~beta.186.ga889181a (a889181a) - on blue-2.shared.runners-manager.gitlab.com/default XxUrkriX -Resolving secrets -00:00 -Preparing the "docker+machine" executor -00:06 -Using Docker executor with image registry.gitlab.com/security-products/api-security:2 ... -Starting service registry.example.com/my-target-app:latest ... -Pulling docker image registry.example.com/my-target-app:latest ... -WARNING: Failed to pull image with policy "always": Error response from daemon: Get https://registry.example.com/my-target-app/manifests/latest: unauthorized (manager.go:237:0s) -ERROR: Job failed: failed to pull image "registry.example.com/my-target-app:latest" with specified policies [always]: Error response from daemon: Get https://registry.example.com/my-target-app/manifests/latest: unauthorized (manager.go:237:0s) -``` - -**Error message** - -- In GitLab 15.9 and earlier, `ERROR: Job failed: failed to pull image` followed by `Error response from daemon: Get IMAGE: unauthorized`. - -**Solution** - -Authentication credentials are provided using the methods outlined in the [Access an image from a private container registry](../../../ci/docker/using_docker_images.md#access-an-image-from-a-private-container-registry) documentation section. The method used is dictated by your container registry provider and its configuration. If your using a container registry provided by a 3rd party, such as a cloud provider (Azure, Google Could (GCP), AWS and so on), check the providers documentation for information on how to authenticate to their container registries. - -The following example uses the [statically defined credentials](../../../ci/docker/using_docker_images.md#use-statically-defined-credentials) authentication method. In this example the container registry is `registry.example.com` and image is `my-target-app:latest`. - -1. Read how to [Determine your `DOCKER_AUTH_CONFIG` data](../../../ci/docker/using_docker_images.md#determine-your-docker_auth_config-data) to understand how to compute the variable value for `DOCKER_AUTH_CONFIG`. The configuration variable `DOCKER_AUTH_CONFIG` contains the Docker JSON configuration to provide the appropriate authentication information. For example, to access private container registry: `registry.example.com` with the credentials `abcdefghijklmn`, the Docker JSON looks like: - - ```json - { - "auths": { - "registry.example.com": { - "auth": "abcdefghijklmn" - } - } - } - ``` - -1. Add the `DOCKER_AUTH_CONFIG` as a CI/CD variable. Instead of adding the configuration variable directly in your `.gitlab-ci.yml`file you should create a project [CI/CD variable](../../../ci/variables/index.md#for-a-project). -1. Rerun your job, and the statically-defined credentials are now used to sign in to the private container registry `registry.example.com`, and let you pull the image `my-target-app:latest`. If succeeded the job console shows an output like: - - ```log - Running with gitlab-runner 15.6.0~beta.186.ga889181a (a889181a) - on blue-4.shared.runners-manager.gitlab.com/default J2nyww-s - Resolving secrets - 00:00 - Preparing the "docker+machine" executor - 00:56 - Using Docker executor with image registry.gitlab.com/security-products/api-security:2 ... - Starting service registry.example.com/my-target-app:latest ... - Authenticating with credentials from $DOCKER_AUTH_CONFIG - Pulling docker image registry.example.com/my-target-app:latest ... - Using docker image sha256:139c39668e5e4417f7d0eb0eeb74145ba862f4f3c24f7c6594ecb2f82dc4ad06 for registry.example.com/my-target-app:latest with digest registry.example.com/my-target- - app@sha256:2b69fc7c3627dbd0ebaa17674c264fcd2f2ba21ed9552a472acf8b065d39039c ... - Waiting for services to be up and running (timeout 30 seconds)... - ``` + + + + \ No newline at end of file diff --git a/doc/user/packages/package_registry/dependency_proxy/index.md b/doc/user/packages/package_registry/dependency_proxy/index.md index 060ff3461e9..c7d08185619 100644 --- a/doc/user/packages/package_registry/dependency_proxy/index.md +++ b/doc/user/packages/package_registry/dependency_proxy/index.md @@ -90,7 +90,7 @@ The dependency proxy uses the [same permissions as the package registry](../inde | Internal | Developer | **{check-circle}** Yes | **{check-circle}** Yes | Package file returned from either the cache or the remote registry. The file is published to the cache. | | Private | Anonymous | **{dotted-circle}** No | **{dotted-circle}** No | Request rejected | | Private | Reporter | **{check-circle}** Yes | **{dotted-circle}** No | Package file returned from either the cache or the remote registry. | -| Internal | Developer | **{check-circle}** Yes | **{check-circle}** Yes | Package file returned from either the cache or the remote registry. The file is published to the cache. | +| Private | Developer | **{check-circle}** Yes | **{check-circle}** Yes | Package file returned from either the cache or the remote registry. The file is published to the cache. | At a minimum, any user who can use the dependency proxy can also use the project's package registry. diff --git a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning.rb b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning.rb index 2f3dde2a871..4890a9f8705 100644 --- a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning.rb +++ b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning.rb @@ -15,10 +15,6 @@ module ActiveRecord # -> (build) { where(partition_id: build.partition_id) }, # partition_foreign_key: :partition_id # - # To control the join filter - # def self.use_partition_id_filter? - # Feature.enabled?(...) - # end # end module Partitioning ActiveSupport.on_load(:active_record) do diff --git a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/base.rb b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/base.rb index c4b3528a526..d9b0326e057 100644 --- a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/base.rb +++ b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/base.rb @@ -10,10 +10,6 @@ module ActiveRecord module Partitioning module Base def _query_constraints_hash - constraints_hash = super - - return constraints_hash unless self.class.use_partition_id_filter? - if self.class.query_constraints_list.nil? { @primary_key => id_in_database } else @@ -24,10 +20,6 @@ module ActiveRecord end module ClassMethods - def use_partition_id_filter? - false - end - def query_constraints(*columns_list) raise ArgumentError, "You must specify at least one column to be used in querying" if columns_list.empty? diff --git a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/abstract_reflection.rb b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/abstract_reflection.rb index 7532cd120a5..d539159c94f 100644 --- a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/abstract_reflection.rb +++ b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/abstract_reflection.rb @@ -12,9 +12,7 @@ module ActiveRecord return klass_scope unless respond_to?(:options) partition_foreign_key = options[:partition_foreign_key] - if partition_foreign_key && klass.use_partition_id_filter? - klass_scope.where!(table[:partition_id].eq(foreign_table[partition_foreign_key])) - end + klass_scope.where!(table[:partition_id].eq(foreign_table[partition_foreign_key])) if partition_foreign_key klass_scope end diff --git a/gems/activerecord-gitlab/spec/support/models.rb b/gems/activerecord-gitlab/spec/support/models.rb index 8a1b37d5a75..ee8609e7110 100644 --- a/gems/activerecord-gitlab/spec/support/models.rb +++ b/gems/activerecord-gitlab/spec/support/models.rb @@ -3,10 +3,6 @@ class PartitionedRecord < ActiveRecord::Base self.abstract_class = true - def self.use_partition_id_filter? - true - end - alias_method :reset, :reload end diff --git a/lib/gitlab/cleanup/orphan_job_artifact_final_objects/generate_list.rb b/lib/gitlab/cleanup/orphan_job_artifact_final_objects/generate_list.rb index a5d0d01fac6..33a5b5877ea 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_final_objects/generate_list.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_final_objects/generate_list.rb @@ -13,7 +13,8 @@ module Gitlab PAGINATORS = { google: Gitlab::Cleanup::OrphanJobArtifactFinalObjects::Paginators::Google, - aws: Gitlab::Cleanup::OrphanJobArtifactFinalObjects::Paginators::Aws + aws: Gitlab::Cleanup::OrphanJobArtifactFinalObjects::Paginators::Aws, + azurerm: Gitlab::Cleanup::OrphanJobArtifactFinalObjects::Paginators::Azure }.freeze def initialize(provider: nil, filename: nil, force_restart: false, logger: Gitlab::AppLogger) diff --git a/lib/gitlab/cleanup/orphan_job_artifact_final_objects/paginators/azure.rb b/lib/gitlab/cleanup/orphan_job_artifact_final_objects/paginators/azure.rb new file mode 100644 index 00000000000..4a154bdc1fe --- /dev/null +++ b/lib/gitlab/cleanup/orphan_job_artifact_final_objects/paginators/azure.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + module OrphanJobArtifactFinalObjects + module Paginators + class Azure < BasePaginator + def page_marker_filter_key + :marker + end + + def max_results_filter_key + :max_results + end + + def last_page?(batch) + batch.next_marker.nil? + end + + def get_next_marker(batch) + batch.next_marker + end + end + end + end + end +end diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index 8092b3c72ea..0c7da1d1952 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -102,11 +102,11 @@ module Gitlab end if entry.plural_id_contains_potential_html? - errors << 'plural id ' + common_message + errors << ('plural id ' + common_message) end if entry.translations_contain_potential_html? - errors << 'translation ' + common_message + errors << ('translation ' + common_message) end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 9f4b4aaafe0..74929c28344 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -13,17 +13,16 @@ module Gitlab new(*args, **kwargs).import end - def initialize(importable:, archive_file:, shared:) + def initialize(importable:, archive_file:, shared:, tmpdir: nil) @importable = importable @archive_file = archive_file @shared = shared + @tmpdir = tmpdir end def import - mkdir_p(@shared.export_path) - mkdir_p(@shared.archive_path) + prepare_extraction_dir - clean_extraction_dir!(@shared.export_path) copy_archive wait_for_archived_file do @@ -35,11 +34,22 @@ module Gitlab false ensure remove_import_file - clean_extraction_dir!(@shared.export_path) + clean_extraction_dir!(target_directory) if should_clean_extraction_dir? end private + def prepare_extraction_dir + if @tmpdir + @tmpdir_valid = validate_tmpdir! + else + mkdir_p(@shared.export_path) + mkdir_p(@shared.archive_path) + end + + clean_extraction_dir!(target_directory) + end + # Exponentially sleep until I/O finishes copying the file def wait_for_archived_file MAX_RETRIES.times do |retry_number| @@ -52,9 +62,9 @@ module Gitlab end def decompress_archive - result = untar_zxf(archive: @archive_file, dir: @shared.export_path) + result = untar_zxf(archive: @archive_file, dir: target_directory) - raise ImporterError, "Unable to decompress #{@archive_file} into #{@shared.export_path}" unless result + raise ImporterError, "Unable to decompress #{@archive_file} into #{target_directory}" unless result result end @@ -62,11 +72,23 @@ module Gitlab def copy_archive return if @archive_file - @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @importable)) + @archive_file = File.join(source_directory, Gitlab::ImportExport.export_filename(exportable: @importable)) remote_download_or_download_or_copy_upload end + def source_directory + return @tmpdir if @tmpdir + + @shared.archive_path + end + + def target_directory + return @tmpdir if @tmpdir + + @shared.export_path + end + def remote_download_or_download_or_copy_upload import_export_upload = @importable.import_export_upload @@ -90,7 +112,7 @@ module Gitlab end def remove_import_file - FileUtils.rm_rf(@archive_file) + FileUtils.rm_rf(@archive_file) if @archive_file end def validate_decompressed_archive_size @@ -100,6 +122,21 @@ module Gitlab def size_validator @size_validator ||= DecompressedArchiveSizeValidator.new(archive_path: @archive_file) end + + def validate_tmpdir! + Gitlab::PathTraversal.check_path_traversal!(@tmpdir) + Gitlab::PathTraversal.check_allowed_absolute_path!(@tmpdir, [Dir.tmpdir]) + + true + end + + attr_reader :tmpdir_valid + + def should_clean_extraction_dir? + return true if @tmpdir.nil? + + tmpdir_valid + end end end end diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index fe41325dfde..bb84de94363 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -230,7 +230,7 @@ module Gitlab def compute_relative_position return unless max_relative_position - max_relative_position + (@relation_index + 1) * Gitlab::RelativePositioning::IDEAL_DISTANCE + max_relative_position + ((@relation_index + 1) * Gitlab::RelativePositioning::IDEAL_DISTANCE) end def max_relative_position diff --git a/lib/gitlab/memory/instrumentation.rb b/lib/gitlab/memory/instrumentation.rb index 4e93d6a9ece..03d6dd9a567 100644 --- a/lib/gitlab/memory/instrumentation.rb +++ b/lib/gitlab/memory/instrumentation.rb @@ -52,7 +52,7 @@ module Gitlab [KEY_MAPPING.fetch(key), current[key].to_i - value] end - result[:mem_total_bytes] = result[:mem_bytes] + result[:mem_objects] * GC::INTERNAL_CONSTANTS[:RVALUE_SIZE] + result[:mem_total_bytes] = result[:mem_bytes] + (result[:mem_objects] * GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]) result end diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index 0b513f5521e..53755c38964 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -96,7 +96,7 @@ module Gitlab end def prometheus_metrics_enabled_unmemoized - !error? && metrics_folder_present? && Gitlab::CurrentSettings.prometheus_metrics_enabled || false + (!error? && metrics_folder_present? && Gitlab::CurrentSettings.prometheus_metrics_enabled) || false end end end diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb index ce1065c0cb3..fffd172a1b8 100644 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -18,7 +18,7 @@ module Gitlab proxy_start = env['HTTP_GITLAB_WORKHORSE_PROXY_START'].presence if trans && proxy_start # Time in milliseconds since gitlab-workhorse started the request - duration = Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000 + duration = (Time.now.to_f * 1_000) - (proxy_start.to_f / 1_000_000) trans.set(:gitlab_transaction_rails_queue_duration_total, duration) do multiprocess_mode :livesum end diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb index a1c340baf23..b7ef395c932 100644 --- a/lib/gitlab/pagination/gitaly_keyset_pager.rb +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -77,7 +77,7 @@ module Gitlab Gitlab::Pagination::OffsetHeaderBuilder.new( request_context: request_context, per_page: per_page, page: 1, next_page: 2, - total: total, total_pages: total / per_page + 1 + total: total, total_pages: (total / per_page) + 1 ).execute end end diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index e4b195767ea..9a8f4219fb5 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -333,7 +333,7 @@ module Gitlab end def confidential_execution_message - confidential_error_message.presence || _("Made this %{type} confidential.") % { type: target_issuable_name } + confidential_error_message.presence || (_("Made this %{type} confidential.") % { type: target_issuable_name }) end def confidential_error_message diff --git a/lib/gitlab/relative_positioning.rb b/lib/gitlab/relative_positioning.rb index c2a73b7cfe5..41ae4853418 100644 --- a/lib/gitlab/relative_positioning.rb +++ b/lib/gitlab/relative_positioning.rb @@ -3,7 +3,7 @@ module Gitlab module RelativePositioning STEPS = 10 - IDEAL_DISTANCE = 2**(STEPS - 1) + 1 + IDEAL_DISTANCE = (2**(STEPS - 1)) + 1 MIN_POSITION = Gitlab::Database::MIN_INT_VALUE START_POSITION = 0 diff --git a/lib/gitlab/template_parser/parser.rb b/lib/gitlab/template_parser/parser.rb index 157339414c4..f3aca909609 100644 --- a/lib/gitlab/template_parser/parser.rb +++ b/lib/gitlab/template_parser/parser.rb @@ -59,7 +59,7 @@ module Gitlab # non-opening tags (e.g. `{foo}`) as text, but not opening tags # themselves (e.g. `{{`). ( - char.repeat(1) | curly_open >> (curly_open | percent).absent? + char.repeat(1) | (curly_open >> (curly_open | percent).absent?) ).repeat(1) end diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb index ba3176ca6e7..8e7073e386f 100644 --- a/lib/gitlab/tree_summary.rb +++ b/lib/gitlab/tree_summary.rb @@ -6,7 +6,7 @@ module Gitlab include ::MarkupHelper CACHE_EXPIRE_IN = 1.hour - MAX_OFFSET = 2**31 - 1 + MAX_OFFSET = (2**31) - 1 attr_reader :commit, :project, :path, :offset, :limit, :user, :resolved_commits diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 2a5240ae287..1bf8eb3f8bd 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -3092,6 +3092,9 @@ msgstr "" msgid "Add environment" msgstr "" +msgid "Add exclusions" +msgstr "" + msgid "Add existing confidential %{issuableType}" msgstr "" @@ -27434,6 +27437,9 @@ msgstr "" msgid "Integrations|Add an integration" msgstr "" +msgid "Integrations|Add exclusions" +msgstr "" + msgid "Integrations|All details" msgstr "" @@ -27473,6 +27479,9 @@ msgstr "" msgid "Integrations|Configure the scope of notifications." msgstr "" +msgid "Integrations|Confirm %{type} exclusion removal" +msgstr "" + msgid "Integrations|Connection details" msgstr "" @@ -27518,6 +27527,9 @@ msgstr "" msgid "Integrations|Error: You are trying to upload something other than an allowed file." msgstr "" +msgid "Integrations|Exclusions" +msgstr "" + msgid "Integrations|GitLab administrators can set up integrations that all groups and projects inherit and use by default. These integrations apply to all groups and projects that don't already use custom settings. You can override custom settings for a group or project if the settings are necessary at that level. Learn more about %{integrations_link_start}instance-level integration management%{link_end}." msgstr "" @@ -27563,6 +27575,12 @@ msgstr "" msgid "Integrations|Perform common tasks with slash commands." msgstr "" +msgid "Integrations|Project exclusion removed" +msgstr "" + +msgid "Integrations|Projects in this list no longer require commits to be signed." +msgstr "" + msgid "Integrations|Projects using custom settings" msgstr "" @@ -27572,6 +27590,9 @@ msgstr "" msgid "Integrations|Projects using custom settings will not be impacted unless the project owner chooses to use parent level defaults." msgstr "" +msgid "Integrations|Remove exclusion" +msgstr "" + msgid "Integrations|Reset integration?" msgstr "" @@ -27611,6 +27632,9 @@ msgstr "" msgid "Integrations|The slash command verification request has expired. Please run the command again." msgstr "" +msgid "Integrations|There are no exclusions" +msgstr "" + msgid "Integrations|There are no projects using custom settings" msgstr "" @@ -27641,6 +27665,9 @@ msgstr "" msgid "Integrations|You haven't activated any integrations yet." msgstr "" +msgid "Integrations|You're removing an exclusion for %{name}. Are you sure you want to continue?" +msgstr "" + msgid "Integrations|You've activated every integration 🎉" msgstr "" @@ -33872,6 +33899,9 @@ msgstr "" msgid "No %{providerTitle} repositories found" msgstr "" +msgid "No %{title} have been added." +msgstr "" + msgid "No Epic" msgstr "" @@ -42657,6 +42687,9 @@ msgstr "" msgid "Remove epic reference" msgstr "" +msgid "Remove exclusion for %{name}" +msgstr "" + msgid "Remove favicon" msgstr "" @@ -45708,6 +45741,9 @@ msgstr "" msgid "Search templates" msgstr "" +msgid "Search to add %{title}" +msgstr "" + msgid "Search users" msgstr "" @@ -47383,6 +47419,9 @@ msgstr "" msgid "SecurityReports|There was an error creating the issue. Please try again." msgstr "" +msgid "SecurityReports|There was an error creating the merge request" +msgstr "" + msgid "SecurityReports|There was an error creating the merge request." msgstr "" @@ -62139,9 +62178,6 @@ msgid_plural "seats" msgstr[0] "" msgstr[1] "" -msgid "security Reports|There was an error creating the merge request" -msgstr "" - msgid "security policy bot users cannot be added to other projects" msgstr "" diff --git a/spec/components/projects/ml/models_index_component_spec.rb b/spec/components/projects/ml/models_index_component_spec.rb deleted file mode 100644 index 1dab4c3c74a..00000000000 --- a/spec/components/projects/ml/models_index_component_spec.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -require "spec_helper" - -RSpec.describe Projects::Ml::ModelsIndexComponent, type: :component, feature_category: :mlops do - let_it_be(:project) { build_stubbed(:project) } - let_it_be(:user) { project.owner } - let_it_be(:model1) { build_stubbed(:ml_models, :with_latest_version_and_package, project: project) } - let_it_be(:model2) { build_stubbed(:ml_models, project: project) } - let_it_be(:models) { [model1, model2] } - - let(:paginator) do - Class.new do - def initialize(models:) - @models = models - end - - def records = @models - def has_next_page? = true - def has_previous_page? = false - def cursor_for_previous_page = 'abcde' - def cursor_for_next_page = 'defgh' - end.new(models: models) - end - - subject(:component) do - described_class.new(project: project, current_user: user, model_count: 5, paginator: paginator) - end - - describe 'rendered' do - let(:element) { page.find("#js-index-ml-models") } - - context 'when user can write model registry' do - before do - allow(model1).to receive(:version_count).and_return(1) - allow(model2).to receive(:version_count).and_return(0) - render_inline component - end - - it 'renders element with view_model' do - expect(Gitlab::Json.parse(element['data-view-model'])).to eq({ - 'models' => [ - { - 'name' => model1.name, - 'version' => model1.latest_version.version, - 'path' => "/#{project.full_path}/-/ml/models/#{model1.id}", - 'versionPackagePath' => "/#{project.full_path}/-/packages/#{model1.latest_version.package_id}", - 'versionPath' => "/#{project.full_path}/-/ml/models/#{model1.id}/versions/#{model1.latest_version.id}", - 'versionCount' => 1 - }, - { - 'name' => model2.name, - 'path' => "/#{project.full_path}/-/ml/models/#{model2.id}", - 'version' => nil, - 'versionPackagePath' => nil, - 'versionPath' => nil, - 'versionCount' => 0 - } - ], - 'pageInfo' => { - 'hasNextPage' => true, - 'hasPreviousPage' => false, - 'startCursor' => 'abcde', - 'endCursor' => 'defgh' - }, - 'modelCount' => 5, - 'createModelPath' => "/#{project.full_path}/-/ml/models/new", - 'canWriteModelRegistry' => true, - 'mlflowTrackingUrl' => "http://localhost/api/v4/projects/#{project.id}/ml/mlflow/api/2.0/mlflow/" - }) - end - end - - context 'when user cannot write model registry' do - before do - allow(Ability).to receive(:allowed?).and_call_original - allow(Ability).to receive(:allowed?) - .with(user, :write_model_registry, project) - .and_return(false) - - render_inline component - end - - it 'canWriteModelRegistry is false' do - expect(Gitlab::Json.parse(element['data-view-model'])['canWriteModelRegistry']).to eq(false) - end - end - end -end diff --git a/spec/components/projects/ml/show_ml_model_component_spec.rb b/spec/components/projects/ml/show_ml_model_component_spec.rb deleted file mode 100644 index d12692ca792..00000000000 --- a/spec/components/projects/ml/show_ml_model_component_spec.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -require "spec_helper" - -RSpec.describe Projects::Ml::ShowMlModelComponent, type: :component, feature_category: :mlops do - let_it_be(:project) { build_stubbed(:project) } - let_it_be(:model1) do - build_stubbed(:ml_models, :with_latest_version_and_package, project: project, description: "A description") - end - - let_it_be(:experiment) { model1.default_experiment.tap { |e| e.iid = 100 } } - let_it_be(:candidate) { model1.latest_version.candidate.tap { |c| c.iid = 101 } } - let_it_be(:candidates) { Array.new(2) { build_stubbed(:ml_candidates, experiment: experiment) } } - - subject(:component) do - described_class.new(model: model1, current_user: model1.user) - end - - describe 'rendered' do - before do - allow(model1).to receive(:candidates).and_return(candidates) - - render_inline component - end - - it 'renders element with view_model' do - element = page.find("#js-mount-show-ml-model") - - expect(Gitlab::Json.parse(element['data-view-model'])).to eq({ - 'model' => { - 'id' => model1.id, - 'name' => model1.name, - 'path' => "/#{project.full_path}/-/ml/models/#{model1.id}", - 'description' => 'A description', - 'latestVersion' => { - 'version' => model1.latest_version.version, - 'description' => model1.latest_version.description, - 'path' => "/#{project.full_path}/-/ml/models/#{model1.id}/versions/#{model1.latest_version.id}", - 'projectPath' => "/#{project.full_path}", - 'packageId' => model1.latest_version.package_id, - 'candidate' => { - 'info' => { - 'iid' => candidate.iid, - 'eid' => candidate.eid, - 'pathToArtifact' => nil, - 'experimentName' => candidate.experiment.name, - 'pathToExperiment' => "/#{project.full_path}/-/ml/experiments/#{experiment.iid}", - 'status' => 'running', - 'path' => "/#{project.full_path}/-/ml/candidates/#{candidate.iid}", - 'ciJob' => nil - }, - 'metrics' => [], - 'params' => [], - 'metadata' => [] - } - }, - 'versionCount' => 1, - 'candidateCount' => 2 - } - }) - end - end -end diff --git a/spec/components/projects/ml/show_ml_model_version_component_spec.rb b/spec/components/projects/ml/show_ml_model_version_component_spec.rb deleted file mode 100644 index 8c7e40d31a2..00000000000 --- a/spec/components/projects/ml/show_ml_model_version_component_spec.rb +++ /dev/null @@ -1,66 +0,0 @@ -# frozen_string_literal: true - -require "spec_helper" - -RSpec.describe Projects::Ml::ShowMlModelVersionComponent, type: :component, feature_category: :mlops do - let_it_be(:project) { build_stubbed(:project) } - let_it_be(:user) { project.owner } - let_it_be(:model) { build_stubbed(:ml_models, project: project) } - let_it_be(:experiment) do - model.default_experiment.iid = 100 - model.default_experiment - end - - let_it_be(:candidate) do - build_stubbed(:ml_candidates, :with_artifact, experiment: experiment, user: user, project: project, - internal_id: 100) - end - - let_it_be(:version) do - build_stubbed(:ml_model_versions, :with_package, model: model, candidate: candidate, description: 'abc') - end - - subject(:component) do - described_class.new(model_version: version, current_user: user) - end - - describe 'rendered' do - before do - render_inline component - end - - it 'renders element with view_model' do - element = page.find("#js-mount-show-ml-model-version") - - expect(Gitlab::Json.parse(element['data-view-model'])).to eq({ - 'modelVersion' => { - 'id' => version.id, - 'version' => version.version, - 'description' => 'abc', - 'projectPath' => "/#{project.full_path}", - 'path' => "/#{project.full_path}/-/ml/models/#{model.id}/versions/#{version.id}", - 'packageId' => version.package_id, - 'model' => { - 'name' => model.name, - 'path' => "/#{project.full_path}/-/ml/models/#{model.id}" - }, - 'candidate' => { - 'info' => { - 'iid' => candidate.iid, - 'eid' => candidate.eid, - 'pathToArtifact' => "/#{project.full_path}/-/packages/#{candidate.artifact.id}", - 'experimentName' => candidate.experiment.name, - 'pathToExperiment' => "/#{project.full_path}/-/ml/experiments/#{experiment.iid}", - 'status' => 'running', - 'path' => "/#{project.full_path}/-/ml/candidates/#{candidate.iid}", - 'ciJob' => nil - }, - 'metrics' => [], - 'params' => [], - 'metadata' => [] - } - } - }) - end - end -end diff --git a/spec/frontend/__helpers__/tracking_internal_events_helper.js b/spec/frontend/__helpers__/tracking_internal_events_helper.js new file mode 100644 index 00000000000..4fc90929df9 --- /dev/null +++ b/spec/frontend/__helpers__/tracking_internal_events_helper.js @@ -0,0 +1,40 @@ +import { InternalEvents } from '~/tracking'; + +export function useMockInternalEventsTracking() { + let originalSnowplow; + let trackEventSpy; + let disposables = []; + + const bindInternalEventDocument = (parent = document) => { + const dispose = InternalEvents.bindInternalEventDocument(parent); + disposables.push(dispose); + + const triggerEvent = (selectorOrEl, eventName = 'click') => { + const event = new Event(eventName, { bubbles: true }); + const el = + typeof selectorOrEl === 'string' ? parent.querySelector(selectorOrEl) : selectorOrEl; + + el.dispatchEvent(event); + }; + + return { triggerEvent, trackEventSpy }; + }; + + beforeEach(() => { + trackEventSpy = jest.spyOn(InternalEvents, 'trackEvent'); + originalSnowplow = window.snowplow; + window.snowplow = () => {}; + }); + + afterEach(() => { + disposables.forEach((dispose) => { + if (dispose) dispose(); + }); + disposables = []; + window.snowplow = originalSnowplow; + }); + + return { + bindInternalEventDocument, + }; +} diff --git a/spec/frontend/integrations/beyond_identity/components/add_exclusions_drawer_spec.js b/spec/frontend/integrations/beyond_identity/components/add_exclusions_drawer_spec.js new file mode 100644 index 00000000000..fb153f53343 --- /dev/null +++ b/spec/frontend/integrations/beyond_identity/components/add_exclusions_drawer_spec.js @@ -0,0 +1,99 @@ +import { GlDrawer } from '@gitlab/ui'; +import AddExclusionsDrawer from '~/integrations/beyond_identity/components/add_exclusions_drawer.vue'; +import ListSelector from '~/vue_shared/components/list_selector/index.vue'; +import { getContentWrapperHeight } from '~/lib/utils/dom_utils'; +import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; +import { DRAWER_Z_INDEX } from '~/lib/utils/constants'; +import { exclusionsMock } from './mock_data'; + +jest.mock('~/lib/utils/dom_utils', () => ({ + getContentWrapperHeight: jest.fn(), +})); + +describe('AddExclusionsDrawer component', () => { + let wrapper; + + const findTitle = () => wrapper.findByTestId('title'); + const findDrawer = () => wrapper.findComponent(GlDrawer); + const findListSelector = () => wrapper.findComponent(ListSelector); + const findAddExclusionsButton = () => wrapper.findByTestId('add-button'); + + const createComponent = (props) => { + return shallowMountExtended(AddExclusionsDrawer, { + propsData: { isOpen: true, ...props }, + }); + }; + + describe('default behavior', () => { + const mockHeaderHeight = '50px'; + + beforeEach(() => { + getContentWrapperHeight.mockReturnValue(mockHeaderHeight); + wrapper = createComponent(); + }); + + it('configures the drawer with header height and z-index', () => { + expect(findDrawer().props()).toMatchObject({ + headerHeight: mockHeaderHeight, + zIndex: DRAWER_Z_INDEX, + }); + }); + }); + + describe('when closed', () => { + beforeEach(() => { + wrapper = createComponent({ isOpen: false }); + }); + + it('the drawer is not shown', () => { + expect(findDrawer().props('open')).toBe(false); + }); + }); + + describe('when open', () => { + beforeEach(() => { + wrapper = createComponent({ isOpen: true }); + }); + + it('opens the drawer', () => { + expect(findDrawer().props('open')).toBe(true); + }); + + it('renders a title', () => { + expect(findTitle().text()).toEqual('Add exclusions'); + }); + + it('renders a project list selector', () => { + expect(findListSelector().props('type')).toBe('projects'); + }); + + it('renders a button for adding exclusions', () => { + expect(findAddExclusionsButton().exists()).toBe(true); + }); + }); + + describe('when exclusions are selected', () => { + beforeEach(() => { + wrapper = createComponent({ showDrawer: true }); + + findListSelector().vm.$emit('select', exclusionsMock[0]); + findListSelector().vm.$emit('select', exclusionsMock[1]); + }); + + it('adds it to the list of selected exclusions', () => { + expect(findListSelector().props('selectedItems')).toEqual(exclusionsMock); + }); + + describe('when Add exclusions button is clicked', () => { + beforeEach(() => findAddExclusionsButton().vm.$emit('click')); + + it('emits the selected exclusions', () => { + expect(wrapper.emitted('add')).toEqual([[exclusionsMock]]); + }); + + it('clears the list of selected exclusions', () => { + expect(findListSelector().props('selectedItems')).toEqual([]); + }); + }); + }); +}); diff --git a/spec/frontend/integrations/beyond_identity/components/exclusions_list_item_spec.js b/spec/frontend/integrations/beyond_identity/components/exclusions_list_item_spec.js new file mode 100644 index 00000000000..1fb9141c983 --- /dev/null +++ b/spec/frontend/integrations/beyond_identity/components/exclusions_list_item_spec.js @@ -0,0 +1,59 @@ +import { GlButton, GlIcon, GlAvatar } from '@gitlab/ui'; +import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; +import ExclusionsListItem from '~/integrations/beyond_identity/components/exclusions_list_item.vue'; +import { exclusionsMock } from './mock_data'; + +describe('ExclusionsListItem component', () => { + let wrapper; + const exclusion = exclusionsMock[0]; + + const findIcon = () => wrapper.findComponent(GlIcon); + const findAvatar = () => wrapper.findComponent(GlAvatar); + const findByText = (text) => wrapper.findByText(text); + const findFindRemoveButton = () => wrapper.findComponent(GlButton); + + const createComponent = () => + shallowMountExtended(ExclusionsListItem, { propsData: { exclusion } }); + + beforeEach(() => { + wrapper = createComponent(); + }); + + describe('default behavior', () => { + it('renders an icon', () => { + expect(findIcon().props('name')).toBe(exclusion.icon); + }); + + it('renders an avatar', () => { + expect(findAvatar().props()).toMatchObject({ + alt: exclusion.name, + entityName: exclusion.name, + size: 32, + shape: 'rect', + src: exclusion.avatarUrl, + fallbackOnError: true, + }); + }); + + it('renders a name', () => { + expect(findByText(exclusion.name).exists()).toBe(true); + }); + + it('renders a remove button', () => { + expect(findFindRemoveButton().attributes('aria-label')).toBe('Remove'); + + expect(findFindRemoveButton().props()).toMatchObject({ + icon: 'remove', + category: 'tertiary', + }); + }); + }); + + describe('remove button', () => { + it('emits remove event when clicked', () => { + findFindRemoveButton().vm.$emit('click'); + + expect(wrapper.emitted('remove')).toBeDefined(); + }); + }); +}); diff --git a/spec/frontend/integrations/beyond_identity/components/exclusions_list_spec.js b/spec/frontend/integrations/beyond_identity/components/exclusions_list_spec.js new file mode 100644 index 00000000000..875e9f0a6f5 --- /dev/null +++ b/spec/frontend/integrations/beyond_identity/components/exclusions_list_spec.js @@ -0,0 +1,111 @@ +import { nextTick } from 'vue'; +import { GlEmptyState } from '@gitlab/ui'; +import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; +import ExclusionsList from '~/integrations/beyond_identity/components/exclusions_list.vue'; +import AddExclusionsDrawer from '~/integrations/beyond_identity/components/add_exclusions_drawer.vue'; +import ExclusionsTabs from '~/integrations/beyond_identity/components/exclusions_tabs.vue'; +import ExclusionsListItem from '~/integrations/beyond_identity/components/exclusions_list_item.vue'; +import ConfirmRemovalModal from '~/integrations/beyond_identity/components/remove_exclusion_confirmation_modal.vue'; +import showToast from '~/vue_shared/plugins/global_toast'; +import { exclusionsMock } from './mock_data'; + +jest.mock('~/vue_shared/plugins/global_toast'); + +describe('ExclusionsList component', () => { + let wrapper; + + const findTabs = () => wrapper.findComponent(ExclusionsTabs); + const findListItems = () => wrapper.findAllComponents(ExclusionsListItem); + const findConfirmRemoveModal = () => wrapper.findComponent(ConfirmRemovalModal); + const findByText = (text) => wrapper.findByText(text); + const findAddExclusionsButton = () => findByText('Add exclusions'); + const findEmptyState = () => wrapper.findComponent(GlEmptyState); + const findDrawer = () => wrapper.findComponent(AddExclusionsDrawer); + + const createComponent = () => shallowMountExtended(ExclusionsList); + + beforeEach(() => { + wrapper = createComponent(); + }); + + describe('default behavior', () => { + it('renders tabs', () => { + expect(findTabs().exists()).toBe(true); + }); + + it('renders help text', () => { + expect( + findByText('Projects in this list no longer require commits to be signed.').exists(), + ).toBe(true); + }); + + it('renders an Add exclusions button', () => { + expect(findAddExclusionsButton().exists()).toBe(true); + }); + + it('renders an Empty state', () => { + expect(findEmptyState().props('title')).toBe('There are no exclusions'); + }); + + it('does not render an open drawer', () => { + expect(findDrawer().props('isOpen')).toBe(false); + }); + }); + + describe('adding Exclusions', () => { + beforeEach(() => findAddExclusionsButton().vm.$emit('click')); + + it('opens a drawer', () => { + expect(findDrawer().props('isOpen')).toBe(true); + }); + + describe('Exclusions added', () => { + beforeEach(() => findDrawer().vm.$emit('add', exclusionsMock)); + + it('lists the added exclusions, sorted by name', async () => { + await nextTick(); + + expect(findListItems().at(0).props('exclusion')).toMatchObject(exclusionsMock[1]); + expect(findListItems().at(1).props('exclusion')).toMatchObject(exclusionsMock[0]); + }); + + it('closes the drawer', () => { + expect(findDrawer().props('isOpen')).toBe(false); + }); + }); + }); + + describe('removing Exclusions', () => { + beforeEach(async () => { + findAddExclusionsButton().vm.$emit('click'); + findDrawer().vm.$emit('add', exclusionsMock); + await nextTick(); + findListItems().at(1).vm.$emit('remove'); + }); + + it('opens a confirmation modal', () => { + expect(findConfirmRemoveModal().props()).toMatchObject({ + name: 'foo', + type: 'project', + visible: true, + }); + }); + + describe('confirmation modal primary action', () => { + beforeEach(() => findConfirmRemoveModal().vm.$emit('primary')); + + it('removes the exclusion', () => { + expect(findListItems().length).toBe(1); + }); + + it('renders a toast', () => { + expect(showToast).toHaveBeenCalledWith('Project exclusion removed', { + action: { + text: 'Undo', + onClick: expect.any(Function), + }, + }); + }); + }); + }); +}); diff --git a/spec/frontend/integrations/beyond_identity/components/exclusions_tabs_spec.js b/spec/frontend/integrations/beyond_identity/components/exclusions_tabs_spec.js new file mode 100644 index 00000000000..271122cd00f --- /dev/null +++ b/spec/frontend/integrations/beyond_identity/components/exclusions_tabs_spec.js @@ -0,0 +1,39 @@ +import { GlNavItem, GlTabs, GlTab } from '@gitlab/ui'; +import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; +import ExclusionsTabs from '~/integrations/beyond_identity/components/exclusions_tabs.vue'; + +describe('ExclusionsTabs component', () => { + let wrapper; + const editPath = 'path/to/edit'; + + const findTabs = () => wrapper.findComponent(GlTabs); + const findNavItem = () => wrapper.findComponent(GlNavItem); + const findTab = () => wrapper.findComponent(GlTab); + + const createComponent = () => + shallowMountExtended(ExclusionsTabs, { + provide: { + editPath, + }, + stubs: { GlTabs }, + }); + + beforeEach(() => { + wrapper = createComponent(); + }); + + describe('default behavior', () => { + it('renders a tabs component', () => { + expect(findTabs().exists()).toBe(true); + }); + + it('renders a nav item for Settings', () => { + expect(findNavItem().text()).toBe('Settings'); + expect(findNavItem().attributes('href')).toBe(editPath); + }); + + it('renders a tab for Exclusions', () => { + expect(findTab().text()).toBe('Exclusions'); + }); + }); +}); diff --git a/spec/frontend/integrations/beyond_identity/components/mock_data.js b/spec/frontend/integrations/beyond_identity/components/mock_data.js new file mode 100644 index 00000000000..2759d1ce6ca --- /dev/null +++ b/spec/frontend/integrations/beyond_identity/components/mock_data.js @@ -0,0 +1,4 @@ +export const exclusionsMock = [ + { id: 1, name: 'foo', type: 'project', icon: 'project', avatarUrl: 'foo.png' }, + { id: 2, name: 'bar', type: 'project', icon: 'project', avatarUrl: 'bar.png' }, +]; diff --git a/spec/frontend/integrations/beyond_identity/components/remove_exclusion_confirmation_modal_spec.js b/spec/frontend/integrations/beyond_identity/components/remove_exclusion_confirmation_modal_spec.js new file mode 100644 index 00000000000..92e40fca314 --- /dev/null +++ b/spec/frontend/integrations/beyond_identity/components/remove_exclusion_confirmation_modal_spec.js @@ -0,0 +1,40 @@ +import { GlModal, GlSprintf } from '@gitlab/ui'; +import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; +import ConfirmRemovalModal from '~/integrations/beyond_identity/components/remove_exclusion_confirmation_modal.vue'; + +describe('ConfirmRemovalModal component', () => { + let wrapper; + const findModal = () => wrapper.findComponent(GlModal); + + const createComponent = () => + shallowMountExtended(ConfirmRemovalModal, { + propsData: { + visible: true, + name: 'Some project', + type: 'project', + }, + stubs: { GlSprintf }, + }); + + beforeEach(() => { + wrapper = createComponent(); + }); + + describe('default behavior', () => { + it('renders a modal component', () => { + expect(findModal().props()).toMatchObject({ + actionPrimary: { text: 'Remove exclusion', attributes: { variant: 'danger' } }, + actionSecondary: { text: 'Cancel', attributes: { category: 'secondary' } }, + modalId: 'confirm-remove-exclusion', + title: 'Confirm project exclusion removal', + visible: true, + }); + }); + + it('renders body content', () => { + expect(findModal().text()).toBe( + "You're removing an exclusion for Some project. Are you sure you want to continue?", + ); + }); + }); +}); diff --git a/spec/frontend/tracking/internal_events_spec.js b/spec/frontend/tracking/internal_events_spec.js index 5256cd76963..fe73e263da8 100644 --- a/spec/frontend/tracking/internal_events_spec.js +++ b/spec/frontend/tracking/internal_events_spec.js @@ -5,6 +5,7 @@ import { LOAD_INTERNAL_EVENTS_SELECTOR } from '~/tracking/constants'; import * as utils from '~/tracking/utils'; import { Tracker } from '~/tracking/tracker'; import Tracking from '~/tracking'; +import { resetHTMLFixture, setHTMLFixture } from 'helpers/fixtures'; const allowedAdditionalProps = { property: 'value', @@ -26,6 +27,19 @@ Tracker.enabled = jest.fn(); const event = 'TestEvent'; describe('InternalEvents', () => { + beforeEach(() => { + setHTMLFixture(`
`); + }); + + afterEach(() => { + resetHTMLFixture(); + }); + + const findButton = () => document.querySelector('[data-testid="button"]'); + const triggerClick = () => { + findButton().dispatchEvent(new Event('click', { bubbles: true })); + }; + describe('trackEvent', () => { const category = 'TestCategory'; @@ -158,30 +172,53 @@ describe('InternalEvents', () => { }); describe('bindInternalEventDocument', () => { + let disposeBind; + let trackEventSpy; + + beforeEach(() => { + Tracker.enabled.mockReturnValue(true); + trackEventSpy = jest.spyOn(InternalEvents, 'trackEvent'); + }); + + afterEach(() => { + disposeBind?.(); + }); + it('should not bind event handlers if tracker is not enabled', () => { Tracker.enabled.mockReturnValue(false); - const result = InternalEvents.bindInternalEventDocument(); - expect(result).toEqual([]); + + disposeBind = InternalEvents.bindInternalEventDocument(); + + expect(disposeBind).toBe(null); expect(utils.getInternalEventHandlers).not.toHaveBeenCalled(); }); it('should not bind event handlers if already bound', () => { - Tracker.enabled.mockReturnValue(true); - document.internalEventsTrackingBound = true; - const result = InternalEvents.bindInternalEventDocument(); - expect(result).toEqual([]); + disposeBind = InternalEvents.bindInternalEventDocument(); + + utils.getInternalEventHandlers.mockReset(); + + const nextDisposeBind = InternalEvents.bindInternalEventDocument(); + + expect(nextDisposeBind).toBe(null); expect(utils.getInternalEventHandlers).not.toHaveBeenCalled(); }); it('should bind event handlers when not bound yet', () => { - Tracker.enabled.mockReturnValue(true); - document.internalEventsTrackingBound = false; - const addEventListenerMock = jest.spyOn(document, 'addEventListener'); + disposeBind = InternalEvents.bindInternalEventDocument(); - const result = InternalEvents.bindInternalEventDocument(); + triggerClick(); - expect(addEventListenerMock).toHaveBeenCalledWith('click', expect.any(Function)); - expect(result).toEqual({ name: 'click', func: expect.any(Function) }); + expect(trackEventSpy).toHaveBeenCalledWith('', {}); + }); + + it('returns function that disposes listener', () => { + disposeBind = InternalEvents.bindInternalEventDocument(); + disposeBind(); + + triggerClick(); + + expect(trackEventSpy).not.toHaveBeenCalled(); }); }); diff --git a/spec/frontend/vue_shared/components/list_selector/index_spec.js b/spec/frontend/vue_shared/components/list_selector/index_spec.js index 6de9a77582c..a51fd2eb65c 100644 --- a/spec/frontend/vue_shared/components/list_selector/index_spec.js +++ b/spec/frontend/vue_shared/components/list_selector/index_spec.js @@ -2,11 +2,13 @@ import Vue from 'vue'; import VueApollo from 'vue-apollo'; import { GlCard, GlIcon, GlCollapsibleListbox, GlSearchBoxByType } from '@gitlab/ui'; import Api from '~/api'; +import RestApi from '~/rest_api'; import { createAlert } from '~/alert'; import { mountExtended } from 'helpers/vue_test_utils_helper'; import ListSelector from '~/vue_shared/components/list_selector/index.vue'; import UserItem from '~/vue_shared/components/list_selector/user_item.vue'; import GroupItem from '~/vue_shared/components/list_selector/group_item.vue'; +import ProjectItem from '~/vue_shared/components/list_selector/project_item.vue'; import DeployKeyItem from '~/vue_shared/components/list_selector/deploy_key_item.vue'; import groupsAutocompleteQuery from '~/graphql_shared/queries/groups_autocomplete.query.graphql'; import createMockApollo from 'helpers/mock_apollo_helper'; @@ -14,6 +16,14 @@ import waitForPromises from 'helpers/wait_for_promises'; import { USERS_RESPONSE_MOCK, GROUPS_RESPONSE_MOCK } from './mock_data'; jest.mock('~/alert'); +jest.mock('~/rest_api', () => ({ + getProjects: jest.fn().mockResolvedValue({ + data: [ + { name: 'Project 1', id: '1' }, + { name: 'Project 2', id: '2' }, + ], + }), +})); Vue.use(VueApollo); describe('List Selector spec', () => { @@ -36,6 +46,10 @@ describe('List Selector spec', () => { type: 'deployKeys', }; + const PROJECTS_MOCK_PROPS = { + type: 'projects', + }; + const groupsAutocompleteQuerySuccess = jest.fn().mockResolvedValue(GROUPS_RESPONSE_MOCK); const createComponent = async (props) => { @@ -60,6 +74,7 @@ describe('List Selector spec', () => { const findSearchBox = () => wrapper.findComponent(GlSearchBoxByType); const findAllUserComponents = () => wrapper.findAllComponents(UserItem); const findAllGroupComponents = () => wrapper.findAllComponents(GroupItem); + const findAllProjectComponents = () => wrapper.findAllComponents(ProjectItem); const findAllDeployKeyComponents = () => wrapper.findAllComponents(DeployKeyItem); beforeEach(() => { @@ -67,6 +82,14 @@ describe('List Selector spec', () => { jest.spyOn(Api, 'groupMembers').mockResolvedValue({ data: USERS_RESPONSE_MOCK }); }); + describe('empty state', () => { + beforeEach(() => createComponent(USERS_MOCK_PROPS)); + + it('renders an empty placeholder', () => { + expect(wrapper.findByText('No users have been added.').exists()).toBe(true); + }); + }); + describe('Users type', () => { const search = 'foo'; @@ -131,7 +154,6 @@ describe('List Selector spec', () => { it('renders a List box component with the correct props', () => { expect(findSearchResultsDropdown().props()).toMatchObject({ - multiple: true, items: searchResponse, }); }); @@ -142,7 +164,7 @@ describe('List Selector spec', () => { it('emits an event when a search result is selected', () => { const firstSearchResult = searchResponse[0]; - findAllUserComponents().at(0).vm.$emit('select', firstSearchResult.username); + findSearchResultsDropdown().vm.$emit('select', firstSearchResult.username); expect(wrapper.emitted('select')).toEqual([ [{ ...firstSearchResult, text: 'Administrator', value: 'root' }], @@ -214,7 +236,6 @@ describe('List Selector spec', () => { it('renders a dropdown for the search results', () => { expect(findSearchResultsDropdown().props()).toMatchObject({ - multiple: true, items: searchResponse, }); }); @@ -225,7 +246,7 @@ describe('List Selector spec', () => { it('emits an event when a search result is selected', () => { const firstSearchResult = searchResponse[0]; - findAllGroupComponents().at(0).vm.$emit('select', firstSearchResult.name); + findSearchResultsDropdown().vm.$emit('select', firstSearchResult.name); expect(wrapper.emitted('select')).toEqual([ [{ ...firstSearchResult, text: 'Flightjs', value: 'Flightjs' }], @@ -301,4 +322,71 @@ describe('List Selector spec', () => { // https://gitlab.com/gitlab-org/gitlab/-/issues/432494 }); }); + + describe('Projects type', () => { + beforeEach(() => createComponent(PROJECTS_MOCK_PROPS)); + + it('renders a correct title', () => { + expect(findTitle().text()).toContain('Projects'); + }); + + it('renders the correct icon', () => { + expect(findIcon().props('name')).toBe('project'); + }); + + describe('searching', () => { + const searchResponse = [ + { name: 'Project 1', id: '1' }, + { name: 'Project 2', id: '2' }, + ]; + const search = 'Project'; + + const emitSearchInput = async () => { + findSearchBox().vm.$emit('input', search); + await waitForPromises(); + }; + + beforeEach(() => emitSearchInput()); + + it('calls query with correct variables when Search box receives an input', () => { + expect(RestApi.getProjects).toHaveBeenCalledWith(search, { membership: false }); + }); + + it('renders a dropdown for the search results', () => { + expect(findSearchResultsDropdown().props()).toMatchObject({ + items: searchResponse, + }); + }); + + it('renders a project component for each search result', () => { + expect(findAllProjectComponents().length).toBe(searchResponse.length); + }); + }); + + describe('selected items', () => { + const selectedGroup = { name: 'Flightjs' }; + const selectedItems = [selectedGroup]; + beforeEach(() => createComponent({ ...GROUPS_MOCK_PROPS, selectedItems })); + + it('renders a heading with the total selected items', () => { + expect(findTitle().text()).toContain('Groups'); + expect(findTitle().text()).toContain('1'); + }); + + it('renders a group component for each selected item', () => { + expect(findAllGroupComponents().length).toBe(selectedItems.length); + expect(findAllGroupComponents().at(0).props()).toMatchObject({ + data: selectedGroup, + canDelete: true, + }); + }); + + it('emits a delete event when a delete event is emitted from the group component', () => { + const name = 'Flightjs'; + findAllGroupComponents().at(0).vm.$emit('delete', name); + + expect(wrapper.emitted('delete')).toEqual([[name]]); + }); + }); + }); }); diff --git a/spec/frontend/vue_shared/components/list_selector/project_item_spec.js b/spec/frontend/vue_shared/components/list_selector/project_item_spec.js new file mode 100644 index 00000000000..f0b0b241e5b --- /dev/null +++ b/spec/frontend/vue_shared/components/list_selector/project_item_spec.js @@ -0,0 +1,59 @@ +import { GlAvatar } from '@gitlab/ui'; +import { mountExtended } from 'helpers/vue_test_utils_helper'; +import ProjectItem from '~/vue_shared/components/list_selector/project_item.vue'; + +describe('GroupItem spec', () => { + let wrapper; + + const MOCK_PROJECT = { + name: 'Project 1', + avatarUrl: 'some/avatar.jpg', + id: 1, + nameWithNamespace: 'Group 1 / Project 1', + }; + + const createComponent = (props) => { + wrapper = mountExtended(ProjectItem, { + propsData: { + data: MOCK_PROJECT, + ...props, + }, + }); + }; + + const findAvatar = () => wrapper.findComponent(GlAvatar); + const findDeleteButton = () => wrapper.findByRole('button', { fullName: 'Delete Group 1' }); + + beforeEach(() => createComponent()); + + it('renders an Avatar component', () => { + expect(findAvatar().props('size')).toBe(32); + expect(findAvatar().attributes()).toMatchObject({ + src: MOCK_PROJECT.avatarUrl, + alt: MOCK_PROJECT.name, + }); + }); + + it('renders a name and namespace', () => { + expect(wrapper.text()).toContain(MOCK_PROJECT.name); + expect(wrapper.text()).toContain(MOCK_PROJECT.nameWithNamespace); + }); + + it('does not render a delete button by default', () => { + expect(findDeleteButton().exists()).toBe(false); + }); + + describe('Delete button', () => { + beforeEach(() => createComponent({ canDelete: true })); + + it('renders a delete button', () => { + expect(findDeleteButton().props('icon')).toBe('remove'); + }); + + it('emits a delete event if the delete button is clicked', () => { + findDeleteButton().trigger('click'); + + expect(wrapper.emitted('delete')).toEqual([[MOCK_PROJECT.id]]); + }); + }); +}); diff --git a/spec/lib/gitlab/import_export/file_importer_spec.rb b/spec/lib/gitlab/import_export/file_importer_spec.rb index b08b04f55ea..81fcba80bfe 100644 --- a/spec/lib/gitlab/import_export/file_importer_spec.rb +++ b/spec/lib/gitlab/import_export/file_importer_spec.rb @@ -35,7 +35,107 @@ RSpec.describe Gitlab::ImportExport::FileImporter, feature_category: :importers FileUtils.rm_rf(storage_path) end - context 'normal run' do + context 'when tmpdir is provided' do + include AfterNextHelpers + + subject(:perform_import) do + described_class.import( + importable: build(:project, import_export_upload: import_export_upload), + archive_file: archive_file, + shared: shared, + tmpdir: tmpdir + ) + end + + let(:archive_file) { nil } + let(:import_export_upload) { build(:import_export_upload) } + let(:tmpdir) { Dir.mktmpdir } + + shared_examples 'it cleans the target directory' do + it 'cleans the target directory' do + # The extraction directory is cleaned twice: once before extraction and + # again afterwards. + expect_next(described_class).to receive(:clean_extraction_dir!).at_least(:twice) + + perform_import + end + end + + shared_examples 'it does not clean the target directory' do + it 'does not clean the directory' do + expect_next(described_class).not_to receive(:clean_extraction_dir!) + + perform_import + end + end + + context 'when the tmpdir is outside of the system tmp folder' do + let(:tmpdir) { '/etc' } + + it 'records an error and skips the import' do + perform_import + + expect(shared.errors).to include('path [FILTERED] is not allowed') + end + + it_behaves_like 'it does not clean the target directory' + end + + context 'when the tmpdir looks like a path traversal' do + let(:tmpdir) { "#{Dir.tmpdir}../../../stealin/ur/etc/passwd" } + + it 'records an error and skips the import' do + perform_import + + expect(shared.errors).to include('Invalid path') + end + + it_behaves_like 'it does not clean the target directory' + end + + context 'and the archive file is not provided' do + it 'copies the archive to the provided temp dir' do + expected_destination_regex = %r{\A#{tmpdir}/[\w\-]+_export\.tar\.gz\z} + + expect_next(described_class) + .to receive(:download_or_copy_upload) + .with( + import_export_upload.import_file, + a_string_matching(expected_destination_regex), + kind_of(Hash) + ) + + perform_import + end + + it_behaves_like 'it cleans the target directory' + end + + context 'and the archive is provided' do + let(:archive_file) { File.join(Dir.tmpdir, 'file_importer_spec.gz') } + + before do + Zlib::GzipWriter.open(archive_file) do |gz| + gz.write('fake tarball contents') + end + end + + it 'extracts the archive to the provided tmp dir' do + expect_next_instance_of(described_class) do |instance| + expect(instance).to receive(:untar_zxf).with( + archive: archive_file, + dir: tmpdir + ) + end + + perform_import + end + + it_behaves_like 'it cleans the target directory' + end + end + + context 'as normal run' do before do described_class.import(importable: build(:project), archive_file: '', shared: shared) end @@ -65,7 +165,7 @@ RSpec.describe Gitlab::ImportExport::FileImporter, feature_category: :importers end it 'does not change a valid file permissions' do - expect(file_permissions(valid_file)).not_to eq(0000) + expect(file_permissions(valid_file)).not_to eq(0o0000) end it 'creates the file in the right subfolder' do @@ -77,7 +177,7 @@ RSpec.describe Gitlab::ImportExport::FileImporter, feature_category: :importers it 'downloads the file from a remote object storage' do import_export_upload = build(:import_export_upload) - project = build( :project, import_export_upload: import_export_upload) + project = build(:project, import_export_upload: import_export_upload) expect_next(described_class) .to receive(:download_or_copy_upload) @@ -97,7 +197,7 @@ RSpec.describe Gitlab::ImportExport::FileImporter, feature_category: :importers it 'downloads the file from a remote object storage' do file_url = 'https://remote.url/file' import_export_upload = build(:import_export_upload, remote_import_url: file_url) - project = build( :project, import_export_upload: import_export_upload) + project = build(:project, import_export_upload: import_export_upload) expect_next(described_class) .to receive(:download) @@ -112,7 +212,7 @@ RSpec.describe Gitlab::ImportExport::FileImporter, feature_category: :importers end end - context 'error' do + context 'when an error occurs' do subject(:import) { described_class.import(importable: build(:project), archive_file: '', shared: shared) } before do @@ -187,7 +287,7 @@ RSpec.describe Gitlab::ImportExport::FileImporter, feature_category: :importers let(:shared) { Gitlab::ImportExport::Shared.new(project) } let(:filepath) { File.join(Dir.tmpdir, 'file_importer_spec.gz') } - subject { described_class.new(importable: project, archive_file: filepath, shared: shared) } + subject(:file_importer) { described_class.new(importable: project, archive_file: filepath, shared: shared) } before do Zlib::GzipWriter.open(filepath) do |gz| @@ -197,7 +297,7 @@ RSpec.describe Gitlab::ImportExport::FileImporter, feature_category: :importers end it 'returns false and sets an error on shared' do - result = subject.import + result = file_importer.import expect(result).to eq(false) expect(shared.errors.join).to eq('Decompressed archive size validation failed.') @@ -205,6 +305,7 @@ RSpec.describe Gitlab::ImportExport::FileImporter, feature_category: :importers end def setup_files + FileUtils.rm_rf(shared.export_path) FileUtils.mkdir_p("#{shared.export_path}/subfolder/") FileUtils.touch(valid_file) FileUtils.ln_s(valid_file, symlink_file) @@ -212,6 +313,6 @@ RSpec.describe Gitlab::ImportExport::FileImporter, feature_category: :importers FileUtils.ln_s(valid_file, hidden_symlink_file) FileUtils.ln_s(valid_file, evil_symlink_file) FileUtils.ln_s(valid_file, custom_mode_symlink_file) - FileUtils.chmod_R(0000, custom_mode_symlink_file) + FileUtils.chmod_R(0o0000, custom_mode_symlink_file) end end diff --git a/spec/models/concerns/ci/has_status_spec.rb b/spec/models/concerns/ci/has_status_spec.rb index d1c92f886bf..dc8ba67a0b3 100644 --- a/spec/models/concerns/ci/has_status_spec.rb +++ b/spec/models/concerns/ci/has_status_spec.rb @@ -368,6 +368,18 @@ RSpec.describe Ci::HasStatus, feature_category: :continuous_integration do end end + describe '.complete_or_manual' do + subject { CommitStatus.complete_or_manual } + + (described_class::COMPLETED_STATUSES + [:manual]).each do |status| + it_behaves_like 'containing the job', status + end + + described_class::ACTIVE_STATUSES.each do |status| + it_behaves_like 'not containing the job', status + end + end + describe '.waiting_for_resource_or_upcoming' do subject { CommitStatus.waiting_for_resource_or_upcoming } diff --git a/spec/support/rspec_order_todo.yml b/spec/support/rspec_order_todo.yml index 0b71cc7bd6c..d1a7a19b39b 100644 --- a/spec/support/rspec_order_todo.yml +++ b/spec/support/rspec_order_todo.yml @@ -6140,7 +6140,6 @@ - './spec/lib/gitlab/import_export/duration_measuring_spec.rb' - './spec/lib/gitlab/import_export/error_spec.rb' - './spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb' -- './spec/lib/gitlab/import_export/file_importer_spec.rb' - './spec/lib/gitlab/import_export/group/object_builder_spec.rb' - './spec/lib/gitlab/import_export/group/relation_factory_spec.rb' - './spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb' diff --git a/spec/workers/projects/import_export/relation_import_worker_spec.rb b/spec/workers/projects/import_export/relation_import_worker_spec.rb index 28bff08f57a..f06cc257e0e 100644 --- a/spec/workers/projects/import_export/relation_import_worker_spec.rb +++ b/spec/workers/projects/import_export/relation_import_worker_spec.rb @@ -18,6 +18,14 @@ RSpec.describe Projects::ImportExport::RelationImportWorker, feature_category: : end context 'when the import succeeds' do + it 'schedules the relation restoration' do + expect_next_instance_of(Gitlab::ImportExport::Project::RelationTreeRestorer) do |restorer| + expect(restorer).to receive(:restore_single_relation).with(tracker.relation) + end + + perform + end + it 'marks the tracker as finished' do expect { perform }.to change { tracker.reload.finished? }.from(false).to(true) end diff --git a/workhorse/cmd/gitlab-workhorse/gitaly_integration_test.go b/workhorse/cmd/gitlab-workhorse/gitaly_integration_test.go index 9e0f11b0df3..8d27376d10a 100644 --- a/workhorse/cmd/gitlab-workhorse/gitaly_integration_test.go +++ b/workhorse/cmd/gitlab-workhorse/gitaly_integration_test.go @@ -174,32 +174,31 @@ func TestAllowedShallowClone(t *testing.T) { } } -// Disabled because of the broken master -// func TestAllowedPush(t *testing.T) { -// skipUnlessRealGitaly(t) +func TestAllowedPush(t *testing.T) { + skipUnlessRealGitaly(t) -// for _, gitalyAddress := range gitalyAddresses { -// t.Run(gitalyAddress, func(t *testing.T) { -// // Create the repository in the Gitaly server -// apiResponse := realGitalyOkBody(t, gitalyAddress) -// require.NoError(t, ensureGitalyRepository(t, apiResponse)) + for _, gitalyAddress := range gitalyAddresses { + t.Run(gitalyAddress, func(t *testing.T) { + // Create the repository in the Gitaly server + apiResponse := realGitalyOkBody(t, gitalyAddress) + require.NoError(t, ensureGitalyRepository(t, apiResponse)) -// // Prepare the test server and backend -// ts := testAuthServer(t, nil, nil, 200, apiResponse) -// ws := startWorkhorseServer(t, ts.URL) + // Prepare the test server and backend + ts := testAuthServer(t, nil, nil, 200, apiResponse) + ws := startWorkhorseServer(t, ts.URL) -// // Do the git clone -// tmpDir := t.TempDir() -// cloneCmd := exec.Command("git", "clone", fmt.Sprintf("%s/%s", ws.URL, testRepo), tmpDir) -// runOrFail(t, cloneCmd) + // Do the git clone + tmpDir := t.TempDir() + cloneCmd := exec.Command("git", "clone", fmt.Sprintf("%s/%s", ws.URL, testRepo), tmpDir) + runOrFail(t, cloneCmd) -// // Perform the git push -// pushCmd := exec.Command("git", "push", fmt.Sprintf("%s/%s", ws.URL, testRepo), fmt.Sprintf("master:%s", newBranch())) -// pushCmd.Dir = tmpDir -// runOrFail(t, pushCmd) -// }) -// } -// } + // Perform the git push + pushCmd := exec.Command("git", "push", fmt.Sprintf("%s/%s", ws.URL, testRepo), fmt.Sprintf("master:%s", newBranch())) + pushCmd.Dir = tmpDir + runOrFail(t, pushCmd) + }) + } +} func TestAllowedGetGitBlob(t *testing.T) { skipUnlessRealGitaly(t) diff --git a/workhorse/cmd/gitlab-workhorse/main_test.go b/workhorse/cmd/gitlab-workhorse/main_test.go index 5a1bc8ad9a6..9d11d00bd7b 100644 --- a/workhorse/cmd/gitlab-workhorse/main_test.go +++ b/workhorse/cmd/gitlab-workhorse/main_test.go @@ -795,6 +795,10 @@ func gitOkBody(t *testing.T) *api.Response { Repository: gitalypb.Repository{ StorageName: "default", RelativePath: "foo/bar.git", + // Gitaly requires a GlRepository to be set in requests that invoke the hooks. + // Set a placeholder to pass the validation. The value doesn't actually matter + // as the calls into Rails' internal API are disabled in tests. + GlRepository: "placeholder-1", }, } } diff --git a/workhorse/internal/headers/content_headers.go b/workhorse/internal/headers/content_headers.go index 3dd8216eec5..487359925c5 100644 --- a/workhorse/internal/headers/content_headers.go +++ b/workhorse/internal/headers/content_headers.go @@ -45,6 +45,7 @@ const ( dummyFilename = "blob" ) +// SafeContentHeaders determines safe content type and disposition for the given data and content disposition func SafeContentHeaders(data []byte, contentDisposition string) (string, string) { detectedContentType := detectContentType(data) diff --git a/workhorse/internal/headers/headers.go b/workhorse/internal/headers/headers.go index 2ba92959c51..3315b03df0c 100644 --- a/workhorse/internal/headers/headers.go +++ b/workhorse/internal/headers/headers.go @@ -1,3 +1,4 @@ +// Package headers provides functionality related to HTTP headers package headers import ( @@ -5,7 +6,7 @@ import ( "strconv" ) -// Max number of bytes that http.DetectContentType needs to get the content type +// MaxDetectSize defines max number of bytes that http.DetectContentType needs to get the content type // Fixme: Go back to 512 bytes once https://gitlab.com/gitlab-org/gitlab/-/issues/325074 // has been merged const MaxDetectSize = 4096 @@ -24,12 +25,14 @@ const ( GitlabWorkhorseDetectContentTypeHeader = "Gitlab-Workhorse-Detect-Content-Type" ) +// ResponseHeaders contains a list of headers that are checked for presence var ResponseHeaders = []string{ XSendFileHeader, GitlabWorkhorseSendDataHeader, GitlabWorkhorseDetectContentTypeHeader, } +// IsDetectContentTypeHeaderPresent checks if the detect content type header is present in the ResponseWriter func IsDetectContentTypeHeaderPresent(rw http.ResponseWriter) bool { header, err := strconv.ParseBool(rw.Header().Get(GitlabWorkhorseDetectContentTypeHeader)) if err != nil || !header { diff --git a/workhorse/internal/headers/headers_test.go b/workhorse/internal/headers/headers_test.go index 555406ff165..d571652a93b 100644 --- a/workhorse/internal/headers/headers_test.go +++ b/workhorse/internal/headers/headers_test.go @@ -11,14 +11,14 @@ func TestIsDetectContentTypeHeaderPresent(t *testing.T) { rw := httptest.NewRecorder() rw.Header().Del(GitlabWorkhorseDetectContentTypeHeader) - require.Equal(t, false, IsDetectContentTypeHeaderPresent(rw)) + require.False(t, IsDetectContentTypeHeaderPresent(rw)) rw.Header().Set(GitlabWorkhorseDetectContentTypeHeader, "true") - require.Equal(t, true, IsDetectContentTypeHeaderPresent(rw)) + require.True(t, IsDetectContentTypeHeaderPresent(rw)) rw.Header().Set(GitlabWorkhorseDetectContentTypeHeader, "false") - require.Equal(t, false, IsDetectContentTypeHeaderPresent(rw)) + require.False(t, IsDetectContentTypeHeaderPresent(rw)) rw.Header().Set(GitlabWorkhorseDetectContentTypeHeader, "foobar") - require.Equal(t, false, IsDetectContentTypeHeaderPresent(rw)) + require.False(t, IsDetectContentTypeHeaderPresent(rw)) } diff --git a/workhorse/internal/rejectmethods/middleware.go b/workhorse/internal/rejectmethods/middleware.go index 171463979d5..8e2a5eb27c1 100644 --- a/workhorse/internal/rejectmethods/middleware.go +++ b/workhorse/internal/rejectmethods/middleware.go @@ -1,3 +1,4 @@ +// Package rejectmethods provides middleware to reject HTTP requests with unknown methods package rejectmethods import ( diff --git a/workhorse/internal/rejectmethods/middleware_test.go b/workhorse/internal/rejectmethods/middleware_test.go index ee5a3eb0ade..d6cdab45223 100644 --- a/workhorse/internal/rejectmethods/middleware_test.go +++ b/workhorse/internal/rejectmethods/middleware_test.go @@ -25,6 +25,7 @@ func TestNewMiddleware(t *testing.T) { middleware.ServeHTTP(recorder, tmpRequest) result := recorder.Result() + defer func() { _ = result.Body.Close() }() require.Equal(t, http.StatusOK, result.StatusCode) }) @@ -33,10 +34,12 @@ func TestNewMiddleware(t *testing.T) { t.Run("UNKNOWN", func(t *testing.T) { tmpRequest, _ := http.NewRequest("UNKNOWN", "/", nil) recorder := httptest.NewRecorder() + defer func() { _ = recorder.Result().Body.Close() }() middleware.ServeHTTP(recorder, tmpRequest) result := recorder.Result() + defer func() { _ = result.Body.Close() }() require.Equal(t, http.StatusMethodNotAllowed, result.StatusCode) }) diff --git a/workhorse/internal/secret/jwt.go b/workhorse/internal/secret/jwt.go index 70970cfc1b3..a8cd1411ed5 100644 --- a/workhorse/internal/secret/jwt.go +++ b/workhorse/internal/secret/jwt.go @@ -1,3 +1,4 @@ +// Package secret provides functionality for handling JWT tokens package secret import ( @@ -7,9 +8,11 @@ import ( ) var ( + // DefaultClaims specifies the default JWT claims DefaultClaims = jwt.RegisteredClaims{Issuer: "gitlab-workhorse"} ) +// JWTTokenString generates a JWT token string with the provided claims func JWTTokenString(claims jwt.Claims) (string, error) { secretBytes, err := Bytes() if err != nil { diff --git a/workhorse/internal/secret/roundtripper.go b/workhorse/internal/secret/roundtripper.go index 50bf7fff5b8..883a2016f1b 100644 --- a/workhorse/internal/secret/roundtripper.go +++ b/workhorse/internal/secret/roundtripper.go @@ -5,7 +5,7 @@ import ( ) const ( - // This header carries the JWT token for gitlab-rails + // RequestHeader carries the JWT token for gitlab-rails RequestHeader = "Gitlab-Workhorse-Api-Request" ) diff --git a/workhorse/internal/secret/secret.go b/workhorse/internal/secret/secret.go index 664f07a52c0..c93c0cced2a 100644 --- a/workhorse/internal/secret/secret.go +++ b/workhorse/internal/secret/secret.go @@ -19,6 +19,7 @@ var ( theSecret = &sec{} ) +// SetPath sets the path for accessing the secret key func SetPath(path string) { theSecret.Lock() defer theSecret.Unlock() @@ -26,7 +27,7 @@ func SetPath(path string) { theSecret.bytes = nil } -// Lazy access to the HMAC secret key. We must be lazy because if the key +// Bytes provides lazy access to the HMAC secret key. We must be lazy because if the key // is not already there, it will be generated by gitlab-rails, and // gitlab-rails is slow. func Bytes() ([]byte, error) { diff --git a/workhorse/internal/upload/multipart_uploader.go b/workhorse/internal/upload/multipart_uploader.go index eee31bd0597..76957e499b9 100644 --- a/workhorse/internal/upload/multipart_uploader.go +++ b/workhorse/internal/upload/multipart_uploader.go @@ -1,3 +1,9 @@ +/* +Package upload provides middleware for handling file uploads in GitLab Workhorse. + +It includes functionality for processing multipart requests, authorizing uploads, +and preparing file uploads before they are saved. +*/ package upload import ( diff --git a/workhorse/internal/upload/object_storage_preparer.go b/workhorse/internal/upload/object_storage_preparer.go index d237a9ca6bc..a7cb96a8073 100644 --- a/workhorse/internal/upload/object_storage_preparer.go +++ b/workhorse/internal/upload/object_storage_preparer.go @@ -6,6 +6,7 @@ import ( "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/destination" ) +// ObjectStoragePreparer prepares objects for upload to object storage. type ObjectStoragePreparer struct { config config.ObjectStorageConfig credentials config.ObjectStorageCredentials @@ -18,6 +19,7 @@ func NewObjectStoragePreparer(c config.Config) Preparer { return &ObjectStoragePreparer{credentials: c.ObjectStorageCredentials, config: c.ObjectStorageConfig} } +// Prepare prepares objects for upload to object storage. func (p *ObjectStoragePreparer) Prepare(a *api.Response) (*destination.UploadOpts, error) { opts, err := destination.GetOpts(a) if err != nil { diff --git a/workhorse/internal/upload/preparer.go b/workhorse/internal/upload/preparer.go index 4d6d8bd1189..ea9302c99bc 100644 --- a/workhorse/internal/upload/preparer.go +++ b/workhorse/internal/upload/preparer.go @@ -15,8 +15,10 @@ type Preparer interface { Prepare(a *api.Response) (*destination.UploadOpts, error) } +// DefaultPreparer implements the Preparer interface. type DefaultPreparer struct{} +// Prepare prepares the upload options based on the API response. func (s *DefaultPreparer) Prepare(a *api.Response) (*destination.UploadOpts, error) { return destination.GetOpts(a) } diff --git a/workhorse/internal/upload/rewrite.go b/workhorse/internal/upload/rewrite.go index acdeb8b74b7..7f5c8323ddd 100644 --- a/workhorse/internal/upload/rewrite.go +++ b/workhorse/internal/upload/rewrite.go @@ -91,7 +91,7 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, fi // Unfortunately as described in https://github.com/golang/go/issues/54133, // we don't have a good way to determine the actual error cause of NextPart() // without an ugly string comparison. - // Note that io.EOF is treated differently from an unexpected EOF: + // io.EOF is treated differently from an unexpected EOF: // https://github.com/golang/go/blob/69d6c7b8ee62b4db5a8f6399e15f27d47b209a29/src/mime/multipart/multipart.go#L395-L405 if err.Error() == "multipart: NextPart: EOF" { return ErrUnexpectedMultipartEOF @@ -157,7 +157,11 @@ func (rew *rewriter) handleFilePart(r *http.Request, name string, p *multipart.P if err != nil { return err } - defer inputReader.Close() + defer func() { + if err = inputReader.Close(); err != nil { + fmt.Printf("error closing multipart file: %v", err) + } + }() fh, err := destination.Upload(ctx, inputReader, -1, filename, opts) if err != nil { @@ -175,7 +179,9 @@ func (rew *rewriter) handleFilePart(r *http.Request, name string, p *multipart.P } for key, value := range fields { - rew.writer.WriteField(key, value) + if err := rew.writer.WriteField(key, value); err != nil { + return err + } rew.finalizedFields[key] = true } @@ -207,7 +213,7 @@ type fileAuthorizer interface { type eagerAuthorizer struct{ response *api.Response } -func (ea *eagerAuthorizer) AuthorizeFile(r *http.Request) (*api.Response, error) { +func (ea *eagerAuthorizer) AuthorizeFile(_ *http.Request) (*api.Response, error) { return ea.response, nil } diff --git a/workhorse/internal/upload/rewrite_test.go b/workhorse/internal/upload/rewrite_test.go index 145f62ee910..5c0cf58c3fe 100644 --- a/workhorse/internal/upload/rewrite_test.go +++ b/workhorse/internal/upload/rewrite_test.go @@ -51,7 +51,7 @@ func TestImageTypeRecongition(t *testing.T) { require.Equal(t, test.isTIFF, isTIFF(input)) runtime.ReadMemStats(&m) - require.Less(t, m.TotalAlloc-start, uint64(50000), "must take reasonable amount of memory to recognise the type") + require.Less(t, m.TotalAlloc-start, uint64(50000), "must take reasonable amount of memory to recognize the type") }) } } diff --git a/workhorse/internal/upload/saved_file_tracker.go b/workhorse/internal/upload/saved_file_tracker.go index 8bacea4610d..ea321be62d1 100644 --- a/workhorse/internal/upload/saved_file_tracker.go +++ b/workhorse/internal/upload/saved_file_tracker.go @@ -13,11 +13,13 @@ import ( "gitlab.com/gitlab-org/gitlab/workhorse/internal/upload/exif" ) +// SavedFileTracker tracks saved files. type SavedFileTracker struct { Request *http.Request rewrittenFields map[string]string } +// Track adds the localPath of the saved file to the tracker with the provided fieldName. func (s *SavedFileTracker) Track(fieldName string, localPath string) { if s.rewrittenFields == nil { s.rewrittenFields = make(map[string]string) @@ -25,10 +27,13 @@ func (s *SavedFileTracker) Track(fieldName string, localPath string) { s.rewrittenFields[fieldName] = localPath } +// Count returns the number of saved file paths tracked by the SavedFileTracker. func (s *SavedFileTracker) Count() int { return len(s.rewrittenFields) } +// ProcessFile processes the uploaded file and tracks its local path. +// It returns an error if the field name has already been processed. func (s *SavedFileTracker) ProcessFile(_ context.Context, fieldName string, file *destination.FileHandler, _ *multipart.Writer, _ *config.Config) error { if _, ok := s.rewrittenFields[fieldName]; ok { return fmt.Errorf("the %v field has already been processed", fieldName) @@ -38,10 +43,14 @@ func (s *SavedFileTracker) ProcessFile(_ context.Context, fieldName string, file return nil } +// ProcessField is a no-op method that implements the FieldProcessor interface. +// It returns nil to indicate successful processing. func (s *SavedFileTracker) ProcessField(_ context.Context, _ string, _ *multipart.Writer) error { return nil } +// Finalize generates a JWT token containing the rewritten fields and sets it in the request header. +// It returns nil if successful, otherwise it returns an error. func (s *SavedFileTracker) Finalize(_ context.Context) error { if s.rewrittenFields == nil { return nil @@ -57,8 +66,12 @@ func (s *SavedFileTracker) Finalize(_ context.Context) error { return nil } +// Name returns the name of the saved file tracker. func (s *SavedFileTracker) Name() string { return "accelerate" } +// TransformContents is a method that implements the destination.FileHandler interface. +// It transforms the contents of the file if it is an image based on its filename and returns a ReadCloser. +// If the file is not an image, it returns the original reader wrapped in a NopCloser. func (*SavedFileTracker) TransformContents(ctx context.Context, filename string, r io.Reader) (io.ReadCloser, error) { if imageType := exif.FileTypeFromSuffix(filename); imageType != exif.TypeUnknown { return handleExifUpload(ctx, r, filename, imageType) diff --git a/workhorse/internal/upload/saved_file_tracker_test.go b/workhorse/internal/upload/saved_file_tracker_test.go index c1a488b06f9..ac945406377 100644 --- a/workhorse/internal/upload/saved_file_tracker_test.go +++ b/workhorse/internal/upload/saved_file_tracker_test.go @@ -34,7 +34,7 @@ func TestSavedFileTracking(t *testing.T) { require.NoError(t, err) rewrittenFields := token.Claims.(*MultipartClaims).RewrittenFields - require.Equal(t, 1, len(rewrittenFields)) + require.Len(t, rewrittenFields, 1) require.Contains(t, rewrittenFields, "test") }