From 4b1fc3dc32e768499d81ed64ea7ed497c1785c48 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 30 Aug 2022 15:10:02 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .gitlab/CODEOWNERS | 95 +++------------- Gemfile | 2 +- Gemfile.lock | 14 ++- .../components/new_access_token_app.vue | 7 +- .../container_expiration_policy_form.vue | 6 +- app/graphql/types/timelog_type.rb | 2 +- app/policies/issuable_policy.rb | 5 + .../import/provider_repo_serializer.rb | 2 + app/views/notify/new_gpg_key_email.html.haml | 11 +- config/audit_events/types/type_schema.json | 74 +++++++++++++ ...mplates_implicit_jobs_sast_iac_monthly.yml | 26 +++++ ...tes_implicit_security_sast_iac_monthly.yml | 26 +++++ ...emplates_implicit_jobs_sast_iac_weekly.yml | 26 +++++ ...ates_implicit_security_sast_iac_weekly.yml | 26 +++++ data/whats_new/2022082200001_15_03.yml | 6 +- ...2324_replace_issues_authorization_index.rb | 18 +++ db/schema_migrations/20220825142324 | 1 + db/structure.sql | 2 +- doc/api/packages/terraform-modules.md | 6 +- .../ci_data_decay/pipeline_partitioning.md | 14 ++- doc/development/audit_event_guide/index.md | 42 +++++-- doc/development/github_importer.md | 17 ++- doc/user/project/import/github.md | 2 + doc/user/tasks.md | 2 +- doc/user/usage_quotas.md | 2 +- .../importer/events/base_importer.rb | 13 +++ .../importer/events/changed_assignee.rb | 2 +- .../importer/events/changed_label.rb | 7 +- .../importer/events/changed_milestone.rb | 7 +- .../github_import/importer/events/closed.rb | 9 +- .../importer/events/cross_referenced.rb | 2 +- .../github_import/importer/events/renamed.rb | 2 +- .../github_import/importer/events/reopened.rb | 9 +- .../importer/issue_event_importer.rb | 4 - .../single_endpoint_issue_events_importer.rb | 26 +++-- .../single_endpoint_notes_importing.rb | 28 +++-- .../usage_data_counters/hll_redis_counter.rb | 1 + .../known_events/ci_templates.yml | 8 -- lib/tasks/gitlab/usage_data.rake | 13 +++ locale/gitlab.pot | 22 +++- .../components/new_access_token_app_spec.js | 20 +++- spec/graphql/types/timelog_type_spec.rb | 2 +- spec/initializers/load_balancing_spec.rb | 84 ++++++++++++++ .../importer/events/changed_assignee_spec.rb | 46 +++++--- .../importer/events/changed_label_spec.rb | 48 +++++--- .../importer/events/changed_milestone_spec.rb | 49 ++++++--- .../importer/events/closed_spec.rb | 86 +++++++++------ .../importer/events/cross_referenced_spec.rb | 103 ++++++++++-------- .../importer/events/renamed_spec.rb | 52 +++++---- .../importer/events/reopened_spec.rb | 63 +++++++---- .../importer/issue_event_importer_spec.rb | 4 - ...gle_endpoint_issue_events_importer_spec.rb | 74 +++++++++++-- spec/policies/issuable_policy_spec.rb | 54 ++++++++- spec/requests/api/usage_data_queries_spec.rb | 32 ++++++ spec/services/issues/create_service_spec.rb | 58 ++++++---- spec/tasks/gitlab/usage_data_rake_spec.rb | 15 +++ tooling/config/CODEOWNERS.yml | 12 +- 57 files changed, 998 insertions(+), 391 deletions(-) create mode 100644 config/audit_events/types/type_schema.json create mode 100644 config/metrics/counts_28d/20220824065053_p_ci_templates_implicit_jobs_sast_iac_monthly.yml create mode 100644 config/metrics/counts_28d/20220824065057_p_ci_templates_implicit_security_sast_iac_monthly.yml create mode 100644 config/metrics/counts_7d/20220824064309_p_ci_templates_implicit_jobs_sast_iac_weekly.yml create mode 100644 config/metrics/counts_7d/20220824064313_p_ci_templates_implicit_security_sast_iac_weekly.yml create mode 100644 db/post_migrate/20220825142324_replace_issues_authorization_index.rb create mode 100644 db/schema_migrations/20220825142324 create mode 100644 spec/initializers/load_balancing_spec.rb diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index a743a601127..6d99d949a79 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -931,6 +931,14 @@ lib/gitlab/checks/** @proglottis @toon @zj-gitlab /app/models/token_with_iv.rb @gitlab-org/manage/authentication-and-authorization/approvers /app/models/webauthn_registration.rb @gitlab-org/manage/authentication-and-authorization/approvers /app/policies/personal_access_token_policy.rb @gitlab-org/manage/authentication-and-authorization/approvers +/app/serializers/group_access_token_entity.rb @gitlab-org/manage/authentication-and-authorization/approvers +/app/serializers/group_access_token_serializer.rb @gitlab-org/manage/authentication-and-authorization/approvers +/app/serializers/impersonation_access_token_entity.rb @gitlab-org/manage/authentication-and-authorization/approvers +/app/serializers/impersonation_access_token_serializer.rb @gitlab-org/manage/authentication-and-authorization/approvers +/app/serializers/personal_access_token_entity.rb @gitlab-org/manage/authentication-and-authorization/approvers +/app/serializers/personal_access_token_serializer.rb @gitlab-org/manage/authentication-and-authorization/approvers +/app/serializers/project_access_token_entity.rb @gitlab-org/manage/authentication-and-authorization/approvers +/app/serializers/project_access_token_serializer.rb @gitlab-org/manage/authentication-and-authorization/approvers /app/services/access_token_validation_service.rb @gitlab-org/manage/authentication-and-authorization/approvers /app/services/auth/ @gitlab-org/manage/authentication-and-authorization/approvers /app/services/authorized_project_update/ @gitlab-org/manage/authentication-and-authorization/approvers @@ -941,8 +949,11 @@ lib/gitlab/checks/** @proglottis @toon @zj-gitlab /app/services/todos/destroy/unauthorized_features_service.rb @gitlab-org/manage/authentication-and-authorization/approvers /app/services/users/authorized_build_service.rb @gitlab-org/manage/authentication-and-authorization/approvers /app/services/users/authorized_create_service.rb @gitlab-org/manage/authentication-and-authorization/approvers +/app/services/users/email_verification/generate_token_service.rb @gitlab-org/manage/authentication-and-authorization/approvers +/app/services/users/email_verification/validate_token_service.rb @gitlab-org/manage/authentication-and-authorization/approvers /app/services/users/refresh_authorized_projects_service.rb @gitlab-org/manage/authentication-and-authorization/approvers /app/services/webauthn/ @gitlab-org/manage/authentication-and-authorization/approvers +/app/validators/json_schemas/build_metadata_id_tokens.json @gitlab-org/manage/authentication-and-authorization/approvers /app/validators/json_schemas/cluster_agent_authorization_configuration.json @gitlab-org/manage/authentication-and-authorization/approvers /app/views/admin/application_settings/_external_authorization_service_form.html.haml @gitlab-org/manage/authentication-and-authorization/approvers /app/views/admin/impersonation_tokens/ @gitlab-org/manage/authentication-and-authorization/approvers @@ -987,20 +998,6 @@ lib/gitlab/checks/** @proglottis @toon @zj-gitlab /app/workers/authorized_project_update/ @gitlab-org/manage/authentication-and-authorization/approvers /app/workers/authorized_projects_worker.rb @gitlab-org/manage/authentication-and-authorization/approvers /app/workers/personal_access_tokens/ @gitlab-org/manage/authentication-and-authorization/approvers -/config/feature_flags/development/access_token_pagination.yml @gitlab-org/manage/authentication-and-authorization/approvers -/config/feature_flags/development/application_settings_tokens_optional_encryption.yml @gitlab-org/manage/authentication-and-authorization/approvers -/config/feature_flags/development/enforce_auth_checks_on_uploads.yml @gitlab-org/manage/authentication-and-authorization/approvers -/config/feature_flags/development/forti_authenticator.yml @gitlab-org/manage/authentication-and-authorization/approvers -/config/feature_flags/development/forti_token_cloud.yml @gitlab-org/manage/authentication-and-authorization/approvers -/config/feature_flags/development/groups_tokens_optional_encryption.yml @gitlab-org/manage/authentication-and-authorization/approvers -/config/feature_flags/development/pbkdf2_password_encryption.yml @gitlab-org/manage/authentication-and-authorization/approvers -/config/feature_flags/development/pbkdf2_password_encryption_write.yml @gitlab-org/manage/authentication-and-authorization/approvers -/config/feature_flags/development/projects_tokens_optional_encryption.yml @gitlab-org/manage/authentication-and-authorization/approvers -/config/feature_flags/development/skip_group_share_unlink_auth_refresh.yml @gitlab-org/manage/authentication-and-authorization/approvers -/config/feature_flags/development/specialized_worker_for_group_lock_update_auth_recalculation.yml @gitlab-org/manage/authentication-and-authorization/approvers -/config/feature_flags/development/update_oauth_registration_flow.yml @gitlab-org/manage/authentication-and-authorization/approvers -/config/feature_flags/development/webauthn.yml @gitlab-org/manage/authentication-and-authorization/approvers -/config/feature_flags/ops/block_password_auth_for_saml_users.yml @gitlab-org/manage/authentication-and-authorization/approvers /config/initializers/01_secret_token.rb @gitlab-org/manage/authentication-and-authorization/approvers /config/initializers/devise_dynamic_password_length_validation.rb @gitlab-org/manage/authentication-and-authorization/approvers /config/initializers/devise_password_length.rb.example @gitlab-org/manage/authentication-and-authorization/approvers @@ -1014,6 +1011,7 @@ lib/gitlab/checks/** @proglottis @toon @zj-gitlab /ee/app/assets/javascripts/access_tokens/ @gitlab-org/manage/authentication-and-authorization/approvers /ee/app/assets/javascripts/audit_events/components/tokens/ @gitlab-org/manage/authentication-and-authorization/approvers /ee/app/assets/javascripts/audit_events/token_utils.js @gitlab-org/manage/authentication-and-authorization/approvers +/ee/app/assets/javascripts/batch_comments/ @gitlab-org/manage/authentication-and-authorization/approvers /ee/app/assets/javascripts/groups/settings/components/ @gitlab-org/manage/authentication-and-authorization/approvers /ee/app/assets/javascripts/pages/admin/application_settings/general/components/ @gitlab-org/manage/authentication-and-authorization/approvers /ee/app/assets/javascripts/pages/groups/omniauth_callbacks/ @gitlab-org/manage/authentication-and-authorization/approvers @@ -1101,6 +1099,7 @@ lib/gitlab/checks/** @proglottis @toon @zj-gitlab /lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb @gitlab-org/manage/authentication-and-authorization/approvers /lib/gitlab/chat_name_token.rb @gitlab-org/manage/authentication-and-authorization/approvers /lib/gitlab/ci/pipeline/expression/token.rb @gitlab-org/manage/authentication-and-authorization/approvers +/lib/gitlab/cleanup/unused_personal_access_tokens.rb @gitlab-org/manage/authentication-and-authorization/approvers /lib/gitlab/external_authorization/ @gitlab-org/manage/authentication-and-authorization/approvers /lib/gitlab/external_authorization.rb @gitlab-org/manage/authentication-and-authorization/approvers /lib/gitlab/grape_logging/loggers/token_logger.rb @gitlab-org/manage/authentication-and-authorization/approvers @@ -1125,21 +1124,6 @@ lib/gitlab/checks/** @proglottis @toon @zj-gitlab /app/services/audit_event_service.rb @gitlab-org/manage/compliance /app/services/concerns/audit_event_save_type.rb @gitlab-org/manage/compliance /app/views/profiles/audit_log.html.haml @gitlab-org/manage/compliance -/data/deprecations/14-3-repository-push-audit-events.yml @gitlab-org/manage/compliance -/data/removals/15_0/removal_manage_repository_push_audit_event.yml @gitlab-org/manage/compliance -/db/docs/audit_events.yml @gitlab-org/manage/compliance -/db/docs/audit_events_external_audit_event_destinations.yml @gitlab-org/manage/compliance -/db/docs/audit_events_streaming_headers.yml @gitlab-org/manage/compliance -/db/migrate/20210819185500_create_external_audit_event_destinations_table.rb @gitlab-org/manage/compliance -/db/migrate/20220524141800_create_audit_events_streaming_headers.rb @gitlab-org/manage/compliance -/db/post_migrate/20210331105335_drop_non_partitioned_audit_events.rb @gitlab-org/manage/compliance -/db/post_migrate/20220119094503_populate_audit_event_streaming_verification_token.rb @gitlab-org/manage/compliance -/doc/administration/audit_event_streaming.md @gitlab-org/manage/compliance -/doc/administration/audit_events.md @gitlab-org/manage/compliance -/doc/administration/audit_reports.md @gitlab-org/manage/compliance -/doc/administration/auditor_users.md @gitlab-org/manage/compliance -/doc/api/audit_events.md @gitlab-org/manage/compliance -/doc/api/graphql/audit_report.md @gitlab-org/manage/compliance /ee/app/assets/javascripts/audit_events/components/audit_events_app.vue @gitlab-org/manage/compliance /ee/app/assets/javascripts/audit_events/components/audit_events_export_button.vue @gitlab-org/manage/compliance /ee/app/assets/javascripts/audit_events/components/audit_events_filter.vue @gitlab-org/manage/compliance @@ -1197,58 +1181,5 @@ lib/gitlab/checks/** @proglottis @toon @zj-gitlab /ee/lib/ee/api/entities/audit_event.rb @gitlab-org/manage/compliance /ee/lib/ee/audit/ @gitlab-org/manage/compliance /ee/lib/ee/gitlab/audit/ @gitlab-org/manage/compliance -/ee/spec/controllers/admin/audit_log_reports_controller_spec.rb @gitlab-org/manage/compliance -/ee/spec/controllers/admin/audit_logs_controller_spec.rb @gitlab-org/manage/compliance -/ee/spec/controllers/groups/audit_events_controller_spec.rb @gitlab-org/manage/compliance -/ee/spec/controllers/projects/audit_events_controller_spec.rb @gitlab-org/manage/compliance -/ee/spec/factories/audit_events/external_audit_event_destinations.rb @gitlab-org/manage/compliance -/ee/spec/features/admin/admin_audit_logs_spec.rb @gitlab-org/manage/compliance -/ee/spec/features/groups/audit_events_spec.rb @gitlab-org/manage/compliance -/ee/spec/features/projects/audit_events_spec.rb @gitlab-org/manage/compliance -/ee/spec/finders/audit_event_finder_spec.rb @gitlab-org/manage/compliance -/ee/spec/fixtures/api/schemas/public_api/v4/audit_event.json @gitlab-org/manage/compliance -/ee/spec/fixtures/api/schemas/public_api/v4/audit_events.json @gitlab-org/manage/compliance -/ee/spec/frontend/audit_events/components/__snapshots__/ @gitlab-org/manage/compliance -/ee/spec/frontend/audit_events/components/audit_events_app_spec.js @gitlab-org/manage/compliance -/ee/spec/frontend/audit_events/components/audit_events_export_button_spec.js @gitlab-org/manage/compliance -/ee/spec/frontend/audit_events/components/audit_events_filter_spec.js @gitlab-org/manage/compliance -/ee/spec/frontend/audit_events/components/audit_events_logs_spec.js @gitlab-org/manage/compliance -/ee/spec/frontend/audit_events/components/audit_events_stream_spec.js @gitlab-org/manage/compliance -/ee/spec/frontend/audit_events/components/audit_events_table_spec.js @gitlab-org/manage/compliance -/ee/spec/frontend/audit_events/components/tokens/shared/ @gitlab-org/manage/compliance -/ee/spec/graphql/types/audit_events/exterrnal_audit_event_destination_type_spec.rb @gitlab-org/manage/compliance -/ee/spec/helpers/audit_events_helper_spec.rb @gitlab-org/manage/compliance -/ee/spec/lib/audit/external_status_check_changes_auditor_spec.rb @gitlab-org/manage/compliance -/ee/spec/lib/audit/group_merge_request_approval_setting_changes_auditor_spec.rb @gitlab-org/manage/compliance -/ee/spec/lib/audit/group_push_rules_changes_auditor_spec.rb @gitlab-org/manage/compliance -/ee/spec/lib/ee/audit/ @gitlab-org/manage/compliance -/ee/spec/lib/gitlab/audit/auditor_spec.rb @gitlab-org/manage/compliance -/ee/spec/models/audit_events/external_audit_event_destination_spec.rb @gitlab-org/manage/compliance -/ee/spec/models/concerns/auditable_spec.rb @gitlab-org/manage/compliance -/ee/spec/models/ee/audit_event_spec.rb @gitlab-org/manage/compliance -/ee/spec/presenters/audit_event_presenter_spec.rb @gitlab-org/manage/compliance -/ee/spec/requests/admin/audit_events_spec.rb @gitlab-org/manage/compliance -/ee/spec/requests/api/audit_events_spec.rb @gitlab-org/manage/compliance -/ee/spec/requests/api/graphql/group/external_audit_event_destinations_spec.rb @gitlab-org/manage/compliance -/ee/spec/requests/groups/audit_events_spec.rb @gitlab-org/manage/compliance -/ee/spec/requests/projects/audit_events_spec.rb @gitlab-org/manage/compliance -/ee/spec/serializers/audit_event_entity_spec.rb @gitlab-org/manage/compliance -/ee/spec/serializers/audit_event_serializer_spec.rb @gitlab-org/manage/compliance -/ee/spec/services/audit_event_service_spec.rb @gitlab-org/manage/compliance -/ee/spec/support/shared_contexts/audit_event_not_licensed_shared_context.rb @gitlab-org/manage/compliance -/ee/spec/support/shared_contexts/audit_event_queue_shared_context.rb @gitlab-org/manage/compliance -/ee/spec/support/shared_examples/audit/ @gitlab-org/manage/compliance -/ee/spec/support/shared_examples/features/audit_events_filter_shared_examples.rb @gitlab-org/manage/compliance -/ee/spec/support/shared_examples/services/audit_event_logging_shared_examples.rb @gitlab-org/manage/compliance -/ee/spec/workers/audit_events/audit_event_streaming_worker_spec.rb @gitlab-org/manage/compliance /lib/gitlab/audit/auditor.rb @gitlab-org/manage/compliance /lib/gitlab/audit_json_logger.rb @gitlab-org/manage/compliance -/spec/factories/audit_events.rb @gitlab-org/manage/compliance -/spec/lib/gitlab/audit/auditor_spec.rb @gitlab-org/manage/compliance -/spec/migrations/populate_audit_event_streaming_verification_token_spec.rb @gitlab-org/manage/compliance -/spec/models/audit_event_spec.rb @gitlab-org/manage/compliance -/spec/services/audit_event_service_spec.rb @gitlab-org/manage/compliance -/spec/services/concerns/audit_event_save_type_spec.rb @gitlab-org/manage/compliance -/spec/support/shared_examples/sends_git_audit_streaming_event_shared_examples.rb @gitlab-org/manage/compliance -/spec/views/profiles/audit_log.html.haml_spec.rb @gitlab-org/manage/compliance -/vendor/project_templates/hipaa_audit_protocol.tar.gz @gitlab-org/manage/compliance diff --git a/Gemfile b/Gemfile index ff38308d4c9..40f1674ba4d 100644 --- a/Gemfile +++ b/Gemfile @@ -39,7 +39,7 @@ gem 'ruby-saml', '~> 1.13.0' gem 'omniauth', '~> 1.8' gem 'omniauth-auth0', '~> 2.0.0' gem 'omniauth-azure-activedirectory-v2', '~> 1.0' -gem 'omniauth-azure-oauth2', '~> 0.0.9' # See vendor/gems/omniauth-azure-oauth2/README.md +gem 'omniauth-azure-oauth2', '~> 0.0.9', path: 'vendor/gems/omniauth-azure-oauth2' # See gem README.md gem 'omniauth-cas3', '~> 1.1.4', path: 'vendor/gems/omniauth-cas3' # See vendor/gems/omniauth-cas3/README.md gem 'omniauth-dingtalk-oauth2', '~> 1.0' gem 'omniauth-alicloud', '~> 1.0.1' diff --git a/Gemfile.lock b/Gemfile.lock index be6c9289dc7..5bfda48ca3a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -24,6 +24,14 @@ PATH connection_pool (~> 2.0) mail (~> 2.7) +PATH + remote: vendor/gems/omniauth-azure-oauth2 + specs: + omniauth-azure-oauth2 (0.0.10) + jwt (>= 1.0, < 3.0) + omniauth (~> 1.0, < 3) + omniauth-oauth2 (~> 1.4) + PATH remote: vendor/gems/omniauth-cas3 specs: @@ -910,10 +918,6 @@ GEM omniauth-oauth2 (>= 1.5) omniauth-azure-activedirectory-v2 (1.0.0) omniauth-oauth2 (~> 1.7) - omniauth-azure-oauth2 (0.0.10) - jwt (>= 1.0, < 3.0) - omniauth (~> 1.0) - omniauth-oauth2 (~> 1.4) omniauth-dingtalk-oauth2 (1.0.1) omniauth-oauth2 (~> 1.7) omniauth-facebook (4.0.0) @@ -1663,7 +1667,7 @@ DEPENDENCIES omniauth-auth0 (~> 2.0.0) omniauth-authentiq (~> 0.3.3) omniauth-azure-activedirectory-v2 (~> 1.0) - omniauth-azure-oauth2 (~> 0.0.9) + omniauth-azure-oauth2 (~> 0.0.9)! omniauth-cas3 (~> 1.1.4)! omniauth-dingtalk-oauth2 (~> 1.0) omniauth-facebook (~> 4.0.0) diff --git a/app/assets/javascripts/access_tokens/components/new_access_token_app.vue b/app/assets/javascripts/access_tokens/components/new_access_token_app.vue index 4b1dd46e4ce..4098130335a 100644 --- a/app/assets/javascripts/access_tokens/components/new_access_token_app.vue +++ b/app/assets/javascripts/access_tokens/components/new_access_token_app.vue @@ -81,7 +81,12 @@ export default { this.infoAlert = createAlert({ message: this.alertInfoMessage, variant: VARIANT_INFO }); - this.form.reset(); + // Reset all input fields except the datepicker. + this.form.querySelectorAll('input:not([id$=expires_at])').forEach((el) => { + // The form token creation is not controlled by Vue. + el.checked = false; + el.value = ''; + }); }, }, }; diff --git a/app/assets/javascripts/packages_and_registries/settings/project/components/container_expiration_policy_form.vue b/app/assets/javascripts/packages_and_registries/settings/project/components/container_expiration_policy_form.vue index ae2d5f4fbc5..2821e0bd6f3 100644 --- a/app/assets/javascripts/packages_and_registries/settings/project/components/container_expiration_policy_form.vue +++ b/app/assets/javascripts/packages_and_registries/settings/project/components/container_expiration_policy_form.vue @@ -199,7 +199,7 @@ export default {

[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367847) in GitLab 15.4. + +All new audit events must have a type definition stored in `config/audit_events/types/` that contains a single source of truth for every auditable event in GitLab. + +### Add a new audit event type + +To add a new audit event type: + +1. Create a new file in `config/audit_events/types/` with the filename matching the name of the event type. For example, a definition for the event type triggered when a + user is added to a project might be stored in `config/audit_events/types/project_add_user.yml`. +1. Add contents to the file that conform to the [schema](#schema) defined in `config/audit_events/types/type_schema.json`. +1. Ensure that all calls to `Gitlab::Audit::Auditor` use the `name` defined in your file. + +### Schema + +| Field | Required | Description | +| ----- | -------- |--------------| +| `name` | yes | Unique, lowercase and underscored name describing the type of event. Must match the filename. | +| `description` | yes | Human-readable description of how this event is triggered | +| `group` | yes | Name of the group that introduced this audit event. For example, `manage::compliance` | +| `introduced_by_issue` | yes | Issue URL that proposed the addition of this type | +| `introduced_by_mr` | yes | MR URL that added this new type | +| `milestone` | yes | Milestone in which this type was added | +| `saved_to_database` | yes | Indicate whether to persist events to database and JSON logs | +| `streamed` | yes | Indicate that events should be streamed to external services (if configured) | + ## Event streaming All events where the entity is a `Group` or `Project` are recorded in the audit log, and also streamed to one or more diff --git a/doc/development/github_importer.md b/doc/development/github_importer.md index e3bf605638d..047625f3f0e 100644 --- a/doc/development/github_importer.md +++ b/doc/development/github_importer.md @@ -101,7 +101,20 @@ label links in the same worker removes the need for performing a separate crawl through the API data, reducing the number of API calls necessary to import a project. -### 8. Stage::ImportNotesWorker +### 8. Stage::ImportIssueEventsWorker + +This worker imports all issues and pull request events. For every event, we +schedule a job for the `Gitlab::GithubImport::ImportIssueEventWorker` worker. + +We can import both issues and pull request events by single stage because of a specific aspect of the GitHub API. It looks like that under the hood, issues and pull requests +GitHub are stored in a single table. Therefore, they have globally-unique IDs and so: + +- Every pull request is an issue. +- Issues aren't pull requests. + +Therefore, both issues and pull requests have a common API for most related things. + +### 9. Stage::ImportNotesWorker This worker imports regular comments for both issues and pull requests. For every comment, we schedule a job for the @@ -112,7 +125,7 @@ returns comments for both issues and pull requests. This means we have to wait for all issues and pull requests to be imported before we can import regular comments. -### 9. Stage::FinishImportWorker +### 10. Stage::FinishImportWorker This worker completes the import process by performing some housekeeping (such as flushing any caches) and by marking the import as completed. diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index a4de3da0775..3b8004946a4 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -184,6 +184,8 @@ The following items of a project are imported: - Pull request "merged by" information (GitLab.com and GitLab 13.7 and later). - Pull request comments replies in discussions ([GitLab.com and GitLab 14.5 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/336596)). - Diff Notes suggestions ([GitLab.com and GitLab 14.7 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/340624)). +- Issue events and pull requests events. [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/7673) in GitLab 15.4 with `github_importer_issue_events_import` + [feature flag](../../../administration/feature_flags.md) disabled by default. References to pull requests and issues are preserved. Each imported repository maintains visibility level unless that [visibility level is restricted](../../public_access.md#restrict-use-of-public-or-internal-projects), in which case it diff --git a/doc/user/tasks.md b/doc/user/tasks.md index b08c0151e7b..304f75a007f 100644 --- a/doc/user/tasks.md +++ b/doc/user/tasks.md @@ -53,7 +53,7 @@ Prerequisites: To create a task: -1. In an issue description, in the **Tasks** section, select **Add**. +1. In the issue description, in the **Tasks** section, select **Add**. 1. Enter the task title. 1. Select **Create task**. diff --git a/doc/user/usage_quotas.md b/doc/user/usage_quotas.md index 5260e0396a7..5d5bbd0bb32 100644 --- a/doc/user/usage_quotas.md +++ b/doc/user/usage_quotas.md @@ -115,7 +115,7 @@ The **Storage** tab of the **Usage Quotas** page warns you of the following: ### Excess storage example -The following example describes an excess storage scenario for namespace _Example Company_: +The following example describes an excess storage scenario for a namespace: | Repository | Storage used | Excess storage | Quota | Status | |------------|--------------|----------------|--------|-------------------| diff --git a/lib/gitlab/github_import/importer/events/base_importer.rb b/lib/gitlab/github_import/importer/events/base_importer.rb index 9ab1d916d33..8218acf2bfb 100644 --- a/lib/gitlab/github_import/importer/events/base_importer.rb +++ b/lib/gitlab/github_import/importer/events/base_importer.rb @@ -29,6 +29,19 @@ module Gitlab def issuable_db_id(object) IssuableFinder.new(project, object).database_id end + + def issuable_type(issue_event) + merge_request_event?(issue_event) ? MergeRequest.name : Issue.name + end + + def merge_request_event?(issue_event) + issue_event.issuable_type == MergeRequest.name + end + + def resource_event_belongs_to(issue_event) + belongs_to_key = merge_request_event?(issue_event) ? :merge_request_id : :issue_id + { belongs_to_key => issuable_db_id(issue_event) } + end end end end diff --git a/lib/gitlab/github_import/importer/events/changed_assignee.rb b/lib/gitlab/github_import/importer/events/changed_assignee.rb index c8f6335e4a8..7b18ea807a5 100644 --- a/lib/gitlab/github_import/importer/events/changed_assignee.rb +++ b/lib/gitlab/github_import/importer/events/changed_assignee.rb @@ -19,7 +19,7 @@ module Gitlab def create_note(issue_event, note_body, assigner_id) Note.create!( system: true, - noteable_type: Issue.name, + noteable_type: issuable_type(issue_event), noteable_id: issuable_db_id(issue_event), project: project, author_id: assigner_id, diff --git a/lib/gitlab/github_import/importer/events/changed_label.rb b/lib/gitlab/github_import/importer/events/changed_label.rb index 818a9202745..83130d18db9 100644 --- a/lib/gitlab/github_import/importer/events/changed_label.rb +++ b/lib/gitlab/github_import/importer/events/changed_label.rb @@ -12,13 +12,14 @@ module Gitlab private def create_event(issue_event) - ResourceLabelEvent.create!( - issue_id: issuable_db_id(issue_event), + attrs = { user_id: author_id(issue_event), label_id: label_finder.id_for(issue_event.label_title), action: action(issue_event.event), created_at: issue_event.created_at - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceLabelEvent.create!(attrs) end def label_finder diff --git a/lib/gitlab/github_import/importer/events/changed_milestone.rb b/lib/gitlab/github_import/importer/events/changed_milestone.rb index 3164c041dc3..39b92d88b58 100644 --- a/lib/gitlab/github_import/importer/events/changed_milestone.rb +++ b/lib/gitlab/github_import/importer/events/changed_milestone.rb @@ -17,14 +17,15 @@ module Gitlab private def create_event(issue_event) - ResourceMilestoneEvent.create!( - issue_id: issuable_db_id(issue_event), + attrs = { user_id: author_id(issue_event), created_at: issue_event.created_at, milestone_id: project.milestones.find_by_title(issue_event.milestone_title)&.id, action: action(issue_event.event), state: DEFAULT_STATE - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceMilestoneEvent.create!(attrs) end def action(event_type) diff --git a/lib/gitlab/github_import/importer/events/closed.rb b/lib/gitlab/github_import/importer/events/closed.rb index ca8730d0f27..58d9dbf826c 100644 --- a/lib/gitlab/github_import/importer/events/closed.rb +++ b/lib/gitlab/github_import/importer/events/closed.rb @@ -17,7 +17,7 @@ module Gitlab project_id: project.id, author_id: author_id(issue_event), action: 'closed', - target_type: Issue.name, + target_type: issuable_type(issue_event), target_id: issuable_db_id(issue_event), created_at: issue_event.created_at, updated_at: issue_event.created_at @@ -25,15 +25,16 @@ module Gitlab end def create_state_event(issue_event) - ResourceStateEvent.create!( + attrs = { user_id: author_id(issue_event), - issue_id: issuable_db_id(issue_event), source_commit: issue_event.commit_id, state: 'closed', close_after_error_tracking_resolve: false, close_auto_resolve_prometheus_alert: false, created_at: issue_event.created_at - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceStateEvent.create!(attrs) end end end diff --git a/lib/gitlab/github_import/importer/events/cross_referenced.rb b/lib/gitlab/github_import/importer/events/cross_referenced.rb index 89fc1bdeb09..b56ae186d3c 100644 --- a/lib/gitlab/github_import/importer/events/cross_referenced.rb +++ b/lib/gitlab/github_import/importer/events/cross_referenced.rb @@ -33,7 +33,7 @@ module Gitlab def create_note(issue_event, note_body, user_id) Note.create!( system: true, - noteable_type: Issue.name, + noteable_type: issuable_type(issue_event), noteable_id: issuable_db_id(issue_event), project: project, author_id: user_id, diff --git a/lib/gitlab/github_import/importer/events/renamed.rb b/lib/gitlab/github_import/importer/events/renamed.rb index 96d112b04c6..fb9e08116ba 100644 --- a/lib/gitlab/github_import/importer/events/renamed.rb +++ b/lib/gitlab/github_import/importer/events/renamed.rb @@ -14,7 +14,7 @@ module Gitlab def note_params(issue_event) { noteable_id: issuable_db_id(issue_event), - noteable_type: Issue.name, + noteable_type: issuable_type(issue_event), project_id: project.id, author_id: author_id(issue_event), note: parse_body(issue_event), diff --git a/lib/gitlab/github_import/importer/events/reopened.rb b/lib/gitlab/github_import/importer/events/reopened.rb index b75344bf817..8abeba0777d 100644 --- a/lib/gitlab/github_import/importer/events/reopened.rb +++ b/lib/gitlab/github_import/importer/events/reopened.rb @@ -17,7 +17,7 @@ module Gitlab project_id: project.id, author_id: author_id(issue_event), action: 'reopened', - target_type: Issue.name, + target_type: issuable_type(issue_event), target_id: issuable_db_id(issue_event), created_at: issue_event.created_at, updated_at: issue_event.created_at @@ -25,12 +25,13 @@ module Gitlab end def create_state_event(issue_event) - ResourceStateEvent.create!( + attrs = { user_id: author_id(issue_event), - issue_id: issuable_db_id(issue_event), state: 'reopened', created_at: issue_event.created_at - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceStateEvent.create!(attrs) end end end diff --git a/lib/gitlab/github_import/importer/issue_event_importer.rb b/lib/gitlab/github_import/importer/issue_event_importer.rb index ef456e56ee1..a5cb71ff6cc 100644 --- a/lib/gitlab/github_import/importer/issue_event_importer.rb +++ b/lib/gitlab/github_import/importer/issue_event_importer.rb @@ -15,11 +15,7 @@ module Gitlab @client = client end - # TODO: Add MergeRequest events support - # https://gitlab.com/groups/gitlab-org/-/epics/7673 def execute - return if issue_event.issuable_type == 'MergeRequest' - importer = event_importer_class(issue_event) if importer importer.new(project, client).execute(issue_event) diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb index 8e4015acbbc..8a9ddfc6ec0 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -7,7 +7,7 @@ module Gitlab include ParallelScheduling include SingleEndpointNotesImporting - PROCESSED_PAGE_CACHE_KEY = 'issues/%{issue_iid}/%{collection}' + PROCESSED_PAGE_CACHE_KEY = 'issues/%{issuable_iid}/%{collection}' BATCH_SIZE = 100 def initialize(project, client, parallel: true) @@ -27,12 +27,20 @@ module Gitlab Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - associated.issue = { 'number' => parent_record.iid } + pull_request = parent_record.is_a? MergeRequest + associated.issue = { 'number' => parent_record.iid, 'pull_request' => pull_request } yield(associated) mark_as_imported(associated) end + # In Github Issues and MergeRequests uses the same API to get their events. + # Even more - they have commonly uniq iid + def each_associated_page(&block) + issues_collection.each_batch(of: BATCH_SIZE, column: :iid) { |batch| process_batch(batch, &block) } + merge_requests_collection.each_batch(of: BATCH_SIZE, column: :iid) { |batch| process_batch(batch, &block) } + end + def importer_class IssueEventImporter end @@ -53,16 +61,20 @@ module Gitlab :issue_timeline end - def parent_collection + def issues_collection project.issues.where.not(iid: already_imported_parents).select(:id, :iid) # rubocop: disable CodeReuse/ActiveRecord end + def merge_requests_collection + project.merge_requests.where.not(iid: already_imported_parents).select(:id, :iid) # rubocop: disable CodeReuse/ActiveRecord + end + def parent_imported_cache_key "github-importer/issues/#{collection_method}/already-imported/#{project.id}" end - def page_counter_id(issue) - PROCESSED_PAGE_CACHE_KEY % { issue_iid: issue.iid, collection: collection_method } + def page_counter_id(issuable) + PROCESSED_PAGE_CACHE_KEY % { issuable_iid: issuable.iid, collection: collection_method } end def id_for_already_imported_cache(event) @@ -74,10 +86,10 @@ module Gitlab end # Cross-referenced events on Github doesn't have id. - def compose_associated_id!(issue, event) + def compose_associated_id!(issuable, event) return if event.event != 'cross-referenced' - event.id = "cross-reference##{issue.id}-in-#{event.source.issue.id}" + event.id = "cross-reference##{issuable.iid}-in-#{event.source.issue.id}" end end end diff --git a/lib/gitlab/github_import/single_endpoint_notes_importing.rb b/lib/gitlab/github_import/single_endpoint_notes_importing.rb index 0a3559adde3..aea4059dfbc 100644 --- a/lib/gitlab/github_import/single_endpoint_notes_importing.rb +++ b/lib/gitlab/github_import/single_endpoint_notes_importing.rb @@ -63,23 +63,27 @@ module Gitlab mark_as_imported(associated) end - def each_associated_page + def each_associated_page(&block) parent_collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch| - batch.each do |parent_record| - # The page counter needs to be scoped by parent_record to avoid skipping - # pages of notes from already imported parent_record. - page_counter = PageCounter.new(project, page_counter_id(parent_record)) - repo = project.import_source - options = collection_options.merge(page: page_counter.current) + process_batch(batch, &block) + end + end - client.each_page(collection_method, repo, parent_record.iid, options) do |page| - next unless page_counter.set(page.number) + def process_batch(batch) + batch.each do |parent_record| + # The page counter needs to be scoped by parent_record to avoid skipping + # pages of notes from already imported parent_record. + page_counter = PageCounter.new(project, page_counter_id(parent_record)) + repo = project.import_source + options = collection_options.merge(page: page_counter.current) - yield parent_record, page - end + client.each_page(collection_method, repo, parent_record.iid, options) do |page| + next unless page_counter.set(page.number) - mark_parent_imported(parent_record) + yield parent_record, page end + + mark_parent_imported(parent_record) end end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 51dc3689dfd..66bbfedc8e3 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -36,6 +36,7 @@ module Gitlab CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS = %w[ ci_users + ci_templates deploy_token_packages error_tracking ide_edit diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index a8f1bab1f20..5da40db3a60 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -99,10 +99,6 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_security_dast_on_demand_api_scan - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - name: p_ci_templates_security_coverage_fuzzing category: ci_templates redis_slot: ci_templates @@ -595,10 +591,6 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_dast_on_demand_api_scan - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - name: p_ci_templates_implicit_security_coverage_fuzzing category: ci_templates redis_slot: ci_templates diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index 9f064ef4c0c..ab06ce15235 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -51,6 +51,19 @@ namespace :gitlab do File.write(Gitlab::UsageDataCounters::CiTemplateUniqueCounter::KNOWN_EVENTS_FILE_PATH, banner + YAML.dump(all_includes).gsub(/ *$/m, '')) end + desc 'GitLab | UsageDataMetrics | Generate raw SQL metrics queries for RSpec' + task generate_sql_metrics_queries: :environment do + path = Rails.root.join('tmp', 'test') + + queries = Timecop.freeze(2021, 1, 1) do + Gitlab::Usage::ServicePingReport.for(output: :metrics_queries) + end + + FileUtils.mkdir_p(path) + FileUtils.chdir(path) + File.write('sql_metrics_queries.json', Gitlab::Json.pretty_generate(queries)) + end + def ci_template_includes_hash(source, template_directory = nil) Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_templates("lib/gitlab/ci/templates/#{template_directory}").map do |template| expanded_template_name = Gitlab::UsageDataCounters::CiTemplateUniqueCounter.expand_template_name("#{template_directory}/#{template}") diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 81c5fb0245a..bb0075b63eb 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -11810,9 +11810,6 @@ msgstr "" msgid "DastConfig|Not enabled" msgstr "" -msgid "DastProfiles| Profile is currently in-use" -msgstr "" - msgid "DastProfiles|A passive scan monitors all HTTP messages (requests and responses) sent to the target. An active scan attacks the target to find potential vulnerabilities." msgstr "" @@ -11990,10 +11987,10 @@ msgstr "" msgid "DastProfiles|Password form field" msgstr "" -msgid "DastProfiles|Profile is being used by this on-demand scan" +msgid "DastProfiles|Profile in use and cannot be renamed" msgstr "" -msgid "DastProfiles|Profile is currently in-use" +msgid "DastProfiles|Profile is being used by this on-demand scan" msgstr "" msgid "DastProfiles|Profile name" @@ -26857,6 +26854,9 @@ msgstr "" msgid "Notify|%{mr_highlight}Merge request%{highlight_end} %{mr_link} %{approved_highlight}was approved by%{highlight_end} %{approver_avatar} %{approver_link}" msgstr "" +msgid "Notify|A new GPG key was added to your account:" +msgstr "" + msgid "Notify|Assignee changed from %{fromNames} to %{toNames}" msgstr "" @@ -26872,6 +26872,15 @@ msgstr "" msgid "Notify|CI/CD project settings" msgstr "" +msgid "Notify|Fingerprint: %{fingerprint}" +msgstr "" + +msgid "Notify|Hi %{user}!" +msgstr "" + +msgid "Notify|If this key was added in error, you can remove it under %{removal_link}" +msgstr "" + msgid "Notify|If you no longer wish to use this domain with GitLab Pages, please remove it from your GitLab project and delete any related DNS records." msgstr "" @@ -35255,6 +35264,9 @@ msgstr "" msgid "SecurityOrchestration|This %{namespaceType} does not contain any security policies." msgstr "" +msgid "SecurityOrchestration|This %{namespaceType} is not linked to a security policy project" +msgstr "" + msgid "SecurityOrchestration|This group" msgstr "" diff --git a/spec/frontend/access_tokens/components/new_access_token_app_spec.js b/spec/frontend/access_tokens/components/new_access_token_app_spec.js index 008096f02e9..b25ac4c6aa8 100644 --- a/spec/frontend/access_tokens/components/new_access_token_app_spec.js +++ b/spec/frontend/access_tokens/components/new_access_token_app_spec.js @@ -34,7 +34,14 @@ describe('~/access_tokens/components/new_access_token_app', () => { beforeEach(() => { // NewAccessTokenApp observes a form element - setHTMLFixture(`
`); + setHTMLFixture( + `
+ + + + +
`, + ); createComponent(); }); @@ -93,12 +100,15 @@ describe('~/access_tokens/components/new_access_token_app', () => { }); }); - it('should reset the form', async () => { - const resetSpy = jest.spyOn(wrapper.vm.form, 'reset'); - + it('should reset all input fields except the date', async () => { + expect(document.querySelector('input[type=text][id$=expires_at]').value).toBe('2022-01-01'); + expect(document.querySelector('input[type=text]:not([id$=expires_at])').value).toBe('1'); + expect(document.querySelector('input[type=checkbox]').checked).toBe(true); await triggerSuccess(); - expect(resetSpy).toHaveBeenCalled(); + expect(document.querySelector('input[type=text][id$=expires_at]').value).toBe('2022-01-01'); + expect(document.querySelector('input[type=text]:not([id$=expires_at])').value).toBe(''); + expect(document.querySelector('input[type=checkbox]').checked).toBe(false); }); }); diff --git a/spec/graphql/types/timelog_type_spec.rb b/spec/graphql/types/timelog_type_spec.rb index c897a25d10d..3a26ba89e04 100644 --- a/spec/graphql/types/timelog_type_spec.rb +++ b/spec/graphql/types/timelog_type_spec.rb @@ -7,7 +7,7 @@ RSpec.describe GitlabSchema.types['Timelog'] do it { expect(described_class.graphql_name).to eq('Timelog') } it { expect(described_class).to have_graphql_fields(fields) } - it { expect(described_class).to require_graphql_authorizations(:read_issue) } + it { expect(described_class).to require_graphql_authorizations(:read_issuable) } it { expect(described_class).to expose_permissions_using(Types::PermissionTypes::Timelog) } describe 'user field' do diff --git a/spec/initializers/load_balancing_spec.rb b/spec/initializers/load_balancing_spec.rb new file mode 100644 index 00000000000..d7d0afd9159 --- /dev/null +++ b/spec/initializers/load_balancing_spec.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'load_balancing', :delete, :reestablished_active_record_base do + subject(:initialize_load_balancer) do + load Rails.root.join('config/initializers/load_balancing.rb') + end + + context 'for a clustered puma worker' do + let!(:group) { create(:group, name: 'my group') } + + before do + # Setup host-based load balancing + # Patch in our load balancer config, simply pointing at the test database twice + allow(Gitlab::Database::LoadBalancing::Configuration).to receive(:for_model) do |base_model| + db_host = base_model.connection_pool.db_config.host + + Gitlab::Database::LoadBalancing::Configuration.new(base_model, [db_host, db_host]) + end + + # Pretend we are in clustered environment + allow(Gitlab::Cluster::LifecycleEvents).to receive(:in_clustered_puma?).and_return(true) + + # Stub out middleware call, as not idempotent + allow(Gitlab::Application.instance.middleware).to receive(:use) + end + + after do + # reset load balancing to original state + allow(Gitlab::Database::LoadBalancing::Configuration).to receive(:for_model).and_call_original + allow(Gitlab::Cluster::LifecycleEvents).to receive(:in_clustered_puma?).and_call_original + + load Rails.root.join('config/initializers/load_balancing.rb') + end + + it 'configures load balancer to have two replica hosts' do + initialize_load_balancer + + simulate_puma_worker do + expect(ApplicationRecord.connection.load_balancer.configuration.hosts.size).to eq(2) + expect(Ci::ApplicationRecord.connection.load_balancer.configuration.hosts.size).to eq(2) + end + end + + # We tried using Process.fork for a more realistic simulation + # but run into bugs where GPRC cannot be used before forking processes. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/333184#note_1081658113 + def simulate_puma_worker + # Called in https://github.com/rails/rails/blob/6-1-stable/activerecord/lib/active_record/connection_adapters/pool_config.rb#L73 + ActiveRecord::ConnectionAdapters::PoolConfig.discard_pools! + + # Called in config/puma.rb + Gitlab::Cluster::LifecycleEvents.do_worker_start + + yield + end + + it 'makes a read query successfully' do + # Clear any previous sticky writes + ::Gitlab::Database::LoadBalancing::Session.clear_session + + initialize_load_balancer + + group_name = simulate_puma_worker do + Group.find_by_name('my group').name + end + + expect(group_name).to eq(group.name) + end + + it 'makes a write query successfully' do + initialize_load_balancer + + expect do + simulate_puma_worker do + Group.touch_all + end + + group.reload + end.to change(group, :updated_at) + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/events/changed_assignee_spec.rb b/spec/lib/gitlab/github_import/importer/events/changed_assignee_spec.rb index 2f6f727dc38..d840227992f 100644 --- a/spec/lib/gitlab/github_import/importer/events/changed_assignee_spec.rb +++ b/spec/lib/gitlab/github_import/importer/events/changed_assignee_spec.rb @@ -10,7 +10,7 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::ChangedAssignee do let_it_be(:assigner) { create(:user) } let(:client) { instance_double('Gitlab::GithubImport::Client') } - let(:issue) { create(:issue, project: project) } + let(:issuable) { create(:issue, project: project) } let(:issue_event) do Gitlab::GithubImport::Representation::IssueEvent.from_json_hash( @@ -21,14 +21,14 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::ChangedAssignee do 'created_at' => '2022-04-26 18:30:53 UTC', 'assigner' => { 'id' => assigner.id, 'login' => assigner.username }, 'assignee' => { 'id' => assignee.id, 'login' => assignee.username }, - 'issue' => { 'number' => issue.iid } + 'issue' => { 'number' => issuable.iid, pull_request: issuable.is_a?(MergeRequest) } ) end let(:note_attrs) do { - noteable_id: issue.id, - noteable_type: Issue.name, + noteable_id: issuable.id, + noteable_type: issuable.class.name, project_id: project.id, author_id: assigner.id, system: true, @@ -45,12 +45,12 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::ChangedAssignee do }.stringify_keys end - shared_examples 'new note' do + shared_examples 'create expected notes' do it 'creates expected note' do - expect { importer.execute(issue_event) }.to change { issue.notes.count } + expect { importer.execute(issue_event) }.to change { issuable.notes.count } .from(0).to(1) - expect(issue.notes.last) + expect(issuable.notes.last) .to have_attributes(expected_note_attrs) end @@ -67,10 +67,26 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::ChangedAssignee do end end + shared_examples 'process assigned & unassigned events' do + context 'when importing an assigned event' do + let(:event_type) { 'assigned' } + let(:expected_note_attrs) { note_attrs.merge(note: "assigned to @#{assignee.username}") } + + it_behaves_like 'create expected notes' + end + + context 'when importing an unassigned event' do + let(:event_type) { 'unassigned' } + let(:expected_note_attrs) { note_attrs.merge(note: "unassigned @#{assigner.username}") } + + it_behaves_like 'create expected notes' + end + end + describe '#execute' do before do allow_next_instance_of(Gitlab::GithubImport::IssuableFinder) do |finder| - allow(finder).to receive(:database_id).and_return(issue.id) + allow(finder).to receive(:database_id).and_return(issuable.id) end allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| allow(finder).to receive(:find).with(assignee.id, assignee.username).and_return(assignee.id) @@ -78,18 +94,14 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::ChangedAssignee do end end - context 'when importing an assigned event' do - let(:event_type) { 'assigned' } - let(:expected_note_attrs) { note_attrs.merge(note: "assigned to @#{assignee.username}") } - - it_behaves_like 'new note' + context 'with Issue' do + it_behaves_like 'process assigned & unassigned events' end - context 'when importing an unassigned event' do - let(:event_type) { 'unassigned' } - let(:expected_note_attrs) { note_attrs.merge(note: "unassigned @#{assigner.username}") } + context 'with MergeRequest' do + let(:issuable) { create(:merge_request, source_project: project, target_project: project) } - it_behaves_like 'new note' + it_behaves_like 'process assigned & unassigned events' end end end diff --git a/spec/lib/gitlab/github_import/importer/events/changed_label_spec.rb b/spec/lib/gitlab/github_import/importer/events/changed_label_spec.rb index e21672aa430..4476b4123ee 100644 --- a/spec/lib/gitlab/github_import/importer/events/changed_label_spec.rb +++ b/spec/lib/gitlab/github_import/importer/events/changed_label_spec.rb @@ -9,7 +9,7 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::ChangedLabel do let_it_be(:user) { create(:user) } let(:client) { instance_double('Gitlab::GithubImport::Client') } - let(:issue) { create(:issue, project: project) } + let(:issuable) { create(:issue, project: project) } let!(:label) { create(:label, project: project) } let(:issue_event) do @@ -19,16 +19,14 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::ChangedLabel do 'event' => event_type, 'commit_id' => nil, 'label_title' => label.title, - 'issue_db_id' => issue.id, 'created_at' => '2022-04-26 18:30:53 UTC', - 'issue' => { 'number' => issue.iid } + 'issue' => { 'number' => issuable.iid, pull_request: issuable.is_a?(MergeRequest) } ) end let(:event_attrs) do { user_id: user.id, - issue_id: issue.id, label_id: label.id, created_at: issue_event.created_at }.stringify_keys @@ -36,9 +34,9 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::ChangedLabel do shared_examples 'new event' do it 'creates a new label event' do - expect { importer.execute(issue_event) }.to change { issue.resource_label_events.count } + expect { importer.execute(issue_event) }.to change { issuable.resource_label_events.count } .from(0).to(1) - expect(issue.resource_label_events.last) + expect(issuable.resource_label_events.last) .to have_attributes(expected_event_attrs) end end @@ -46,24 +44,44 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::ChangedLabel do before do allow(Gitlab::Cache::Import::Caching).to receive(:read_integer).and_return(label.id) allow_next_instance_of(Gitlab::GithubImport::IssuableFinder) do |finder| - allow(finder).to receive(:database_id).and_return(issue.id) + allow(finder).to receive(:database_id).and_return(issuable.id) end allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| allow(finder).to receive(:find).with(user.id, user.username).and_return(user.id) end end - context 'when importing a labeled event' do - let(:event_type) { 'labeled' } - let(:expected_event_attrs) { event_attrs.merge(action: 'add') } + context 'with Issue' do + context 'when importing a labeled event' do + let(:event_type) { 'labeled' } + let(:expected_event_attrs) { event_attrs.merge(issue_id: issuable.id, action: 'add') } - it_behaves_like 'new event' + it_behaves_like 'new event' + end + + context 'when importing an unlabeled event' do + let(:event_type) { 'unlabeled' } + let(:expected_event_attrs) { event_attrs.merge(issue_id: issuable.id, action: 'remove') } + + it_behaves_like 'new event' + end end - context 'when importing an unlabeled event' do - let(:event_type) { 'unlabeled' } - let(:expected_event_attrs) { event_attrs.merge(action: 'remove') } + context 'with MergeRequest' do + let(:issuable) { create(:merge_request, source_project: project, target_project: project) } - it_behaves_like 'new event' + context 'when importing a labeled event' do + let(:event_type) { 'labeled' } + let(:expected_event_attrs) { event_attrs.merge(merge_request_id: issuable.id, action: 'add') } + + it_behaves_like 'new event' + end + + context 'when importing an unlabeled event' do + let(:event_type) { 'unlabeled' } + let(:expected_event_attrs) { event_attrs.merge(merge_request_id: issuable.id, action: 'remove') } + + it_behaves_like 'new event' + end end end diff --git a/spec/lib/gitlab/github_import/importer/events/changed_milestone_spec.rb b/spec/lib/gitlab/github_import/importer/events/changed_milestone_spec.rb index 2687627fc23..bc14b81bd91 100644 --- a/spec/lib/gitlab/github_import/importer/events/changed_milestone_spec.rb +++ b/spec/lib/gitlab/github_import/importer/events/changed_milestone_spec.rb @@ -9,7 +9,7 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::ChangedMilestone do let_it_be(:user) { create(:user) } let(:client) { instance_double('Gitlab::GithubImport::Client') } - let(:issue) { create(:issue, project: project) } + let(:issuable) { create(:issue, project: project) } let!(:milestone) { create(:milestone, project: project) } let(:issue_event) do @@ -19,16 +19,15 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::ChangedMilestone do 'event' => event_type, 'commit_id' => nil, 'milestone_title' => milestone.title, - 'issue_db_id' => issue.id, + 'issue_db_id' => issuable.id, 'created_at' => '2022-04-26 18:30:53 UTC', - 'issue' => { 'number' => issue.iid } + 'issue' => { 'number' => issuable.iid, pull_request: issuable.is_a?(MergeRequest) } ) end let(:event_attrs) do { user_id: user.id, - issue_id: issue.id, milestone_id: milestone.id, state: 'opened', created_at: issue_event.created_at @@ -37,9 +36,9 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::ChangedMilestone do shared_examples 'new event' do it 'creates a new milestone event' do - expect { importer.execute(issue_event) }.to change { issue.resource_milestone_events.count } + expect { importer.execute(issue_event) }.to change { issuable.resource_milestone_events.count } .from(0).to(1) - expect(issue.resource_milestone_events.last) + expect(issuable.resource_milestone_events.last) .to have_attributes(expected_event_attrs) end end @@ -48,25 +47,45 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::ChangedMilestone do before do allow(Gitlab::Cache::Import::Caching).to receive(:read_integer).and_return(milestone.id) allow_next_instance_of(Gitlab::GithubImport::IssuableFinder) do |finder| - allow(finder).to receive(:database_id).and_return(issue.id) + allow(finder).to receive(:database_id).and_return(issuable.id) end allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| allow(finder).to receive(:find).with(user.id, user.username).and_return(user.id) end end - context 'when importing a milestoned event' do - let(:event_type) { 'milestoned' } - let(:expected_event_attrs) { event_attrs.merge(action: 'add') } + context 'with Issue' do + context 'when importing a milestoned event' do + let(:event_type) { 'milestoned' } + let(:expected_event_attrs) { event_attrs.merge(issue_id: issuable.id, action: 'add') } - it_behaves_like 'new event' + it_behaves_like 'new event' + end + + context 'when importing demilestoned event' do + let(:event_type) { 'demilestoned' } + let(:expected_event_attrs) { event_attrs.merge(issue_id: issuable.id, action: 'remove') } + + it_behaves_like 'new event' + end end - context 'when importing demilestoned event' do - let(:event_type) { 'demilestoned' } - let(:expected_event_attrs) { event_attrs.merge(action: 'remove') } + context 'with MergeRequest' do + let(:issuable) { create(:merge_request, source_project: project, target_project: project) } - it_behaves_like 'new event' + context 'when importing a milestoned event' do + let(:event_type) { 'milestoned' } + let(:expected_event_attrs) { event_attrs.merge(merge_request_id: issuable.id, action: 'add') } + + it_behaves_like 'new event' + end + + context 'when importing demilestoned event' do + let(:event_type) { 'demilestoned' } + let(:expected_event_attrs) { event_attrs.merge(merge_request_id: issuable.id, action: 'remove') } + + it_behaves_like 'new event' + end end end end diff --git a/spec/lib/gitlab/github_import/importer/events/closed_spec.rb b/spec/lib/gitlab/github_import/importer/events/closed_spec.rb index 9a49d80a8bb..f7e38f373c0 100644 --- a/spec/lib/gitlab/github_import/importer/events/closed_spec.rb +++ b/spec/lib/gitlab/github_import/importer/events/closed_spec.rb @@ -9,7 +9,7 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::Closed do let_it_be(:user) { create(:user) } let(:client) { instance_double('Gitlab::GithubImport::Client') } - let(:issue) { create(:issue, project: project) } + let(:issuable) { create(:issue, project: project) } let(:commit_id) { nil } let(:issue_event) do @@ -21,7 +21,7 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::Closed do 'event' => 'closed', 'created_at' => '2022-04-26 18:30:53 UTC', 'commit_id' => commit_id, - 'issue' => { 'number' => issue.iid } + 'issue' => { 'number' => issuable.iid, pull_request: issuable.is_a?(MergeRequest) } ) end @@ -29,54 +29,74 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::Closed do { project_id: project.id, author_id: user.id, - target_id: issue.id, - target_type: Issue.name, + target_id: issuable.id, + target_type: issuable.class.name, action: 'closed', created_at: issue_event.created_at, updated_at: issue_event.created_at }.stringify_keys end - let(:expected_state_event_attrs) do - { - user_id: user.id, - issue_id: issue.id, - state: 'closed', - created_at: issue_event.created_at - }.stringify_keys - end - before do allow_next_instance_of(Gitlab::GithubImport::IssuableFinder) do |finder| - allow(finder).to receive(:database_id).and_return(issue.id) + allow(finder).to receive(:database_id).and_return(issuable.id) end allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| allow(finder).to receive(:find).with(user.id, user.username).and_return(user.id) end end - it 'creates expected event and state event' do - importer.execute(issue_event) - - expect(issue.events.count).to eq 1 - expect(issue.events[0].attributes) - .to include expected_event_attrs - - expect(issue.resource_state_events.count).to eq 1 - expect(issue.resource_state_events[0].attributes) - .to include expected_state_event_attrs - end - - context 'when closed by commit' do - let!(:closing_commit) { create(:commit, project: project) } - let(:commit_id) { closing_commit.id } - + shared_examples 'new event' do it 'creates expected event and state event' do importer.execute(issue_event) - expect(issue.events.count).to eq 1 - state_event = issue.resource_state_events.last - expect(state_event.source_commit).to eq commit_id[0..40] + expect(issuable.events.count).to eq 1 + expect(issuable.events[0].attributes) + .to include expected_event_attrs + + expect(issuable.resource_state_events.count).to eq 1 + expect(issuable.resource_state_events[0].attributes) + .to include expected_state_event_attrs + end + + context 'when closed by commit' do + let!(:closing_commit) { create(:commit, project: project) } + let(:commit_id) { closing_commit.id } + + it 'creates expected event and state event' do + importer.execute(issue_event) + + expect(issuable.events.count).to eq 1 + state_event = issuable.resource_state_events.last + expect(state_event.source_commit).to eq commit_id[0..40] + end end end + + context 'with Issue' do + let(:expected_state_event_attrs) do + { + user_id: user.id, + issue_id: issuable.id, + state: 'closed', + created_at: issue_event.created_at + }.stringify_keys + end + + it_behaves_like 'new event' + end + + context 'with MergeRequest' do + let(:issuable) { create(:merge_request, source_project: project, target_project: project) } + let(:expected_state_event_attrs) do + { + user_id: user.id, + merge_request_id: issuable.id, + state: 'closed', + created_at: issue_event.created_at + }.stringify_keys + end + + it_behaves_like 'new event' + end end diff --git a/spec/lib/gitlab/github_import/importer/events/cross_referenced_spec.rb b/spec/lib/gitlab/github_import/importer/events/cross_referenced_spec.rb index 68e001c7364..bf19147d4c8 100644 --- a/spec/lib/gitlab/github_import/importer/events/cross_referenced_spec.rb +++ b/spec/lib/gitlab/github_import/importer/events/cross_referenced_spec.rb @@ -9,9 +9,8 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::CrossReferenced, :clean_g let_it_be(:user) { create(:user) } let(:client) { instance_double('Gitlab::GithubImport::Client') } - let(:issue_iid) { 999 } - let(:issue) { create(:issue, project: project, iid: issue_iid) } + let(:issuable) { create(:issue, project: project, iid: issue_iid) } let(:referenced_in) { build_stubbed(:issue, project: project, iid: issue_iid + 1) } let(:commit_id) { nil } @@ -30,7 +29,7 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::CrossReferenced, :clean_g } }, 'created_at' => '2022-04-26 18:30:53 UTC', - 'issue' => { 'number' => issue.iid } + 'issue' => { 'number' => issuable.iid, pull_request: issuable.is_a?(MergeRequest) } ) end @@ -38,8 +37,8 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::CrossReferenced, :clean_g let(:expected_note_attrs) do { system: true, - noteable_type: Issue.name, - noteable_id: issue.id, + noteable_type: issuable.class.name, + noteable_id: issuable.id, project_id: project.id, author_id: user.id, note: expected_note_body, @@ -47,58 +46,70 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::CrossReferenced, :clean_g }.stringify_keys end - context 'when referenced in other issue' do - let(:expected_note_body) { "mentioned in issue ##{referenced_in.iid}" } + shared_examples 'import cross-referenced event' do + context 'when referenced in other issue' do + let(:expected_note_body) { "mentioned in issue ##{referenced_in.iid}" } - before do - allow_next_instance_of(Gitlab::GithubImport::IssuableFinder) do |finder| - allow(finder).to receive(:database_id).and_return(referenced_in.iid) - allow(finder).to receive(:database_id).and_return(issue.id) + before do + allow_next_instance_of(Gitlab::GithubImport::IssuableFinder) do |finder| + allow(finder).to receive(:database_id).and_return(referenced_in.iid) + allow(finder).to receive(:database_id).and_return(issuable.id) + end + allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| + allow(finder).to receive(:find).with(user.id, user.username).and_return(user.id) + end end - allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| - allow(finder).to receive(:find).with(user.id, user.username).and_return(user.id) + + it 'creates expected note' do + importer.execute(issue_event) + + expect(issuable.notes.count).to eq 1 + expect(issuable.notes[0]).to have_attributes expected_note_attrs + expect(issuable.notes[0].system_note_metadata.action).to eq 'cross_reference' end end - it 'creates expected note' do - importer.execute(issue_event) + context 'when referenced in pull request' do + let(:referenced_in) { build_stubbed(:merge_request, project: project) } + let(:pull_request_resource) { { 'id' => referenced_in.iid } } - expect(issue.notes.count).to eq 1 - expect(issue.notes[0]).to have_attributes expected_note_attrs - expect(issue.notes[0].system_note_metadata.action).to eq 'cross_reference' + let(:expected_note_body) { "mentioned in merge request !#{referenced_in.iid}" } + + before do + allow_next_instance_of(Gitlab::GithubImport::IssuableFinder) do |finder| + allow(finder).to receive(:database_id).and_return(referenced_in.iid) + allow(finder).to receive(:database_id).and_return(issuable.id) + end + allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| + allow(finder).to receive(:find).with(user.id, user.username).and_return(user.id) + end + end + + it 'creates expected note' do + importer.execute(issue_event) + + expect(issuable.notes.count).to eq 1 + expect(issuable.notes[0]).to have_attributes expected_note_attrs + expect(issuable.notes[0].system_note_metadata.action).to eq 'cross_reference' + end + end + + context 'when referenced in out of project issue/pull_request' do + it 'does not create expected note' do + importer.execute(issue_event) + + expect(issuable.notes.count).to eq 0 + end end end - context 'when referenced in pull request' do - let(:referenced_in) { build_stubbed(:merge_request, project: project) } - let(:pull_request_resource) { { 'id' => referenced_in.iid } } - - let(:expected_note_body) { "mentioned in merge request !#{referenced_in.iid}" } - - before do - allow_next_instance_of(Gitlab::GithubImport::IssuableFinder) do |finder| - allow(finder).to receive(:database_id).and_return(referenced_in.iid) - allow(finder).to receive(:database_id).and_return(issue.id) - end - allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| - allow(finder).to receive(:find).with(user.id, user.username).and_return(user.id) - end - end - - it 'creates expected note' do - importer.execute(issue_event) - - expect(issue.notes.count).to eq 1 - expect(issue.notes[0]).to have_attributes expected_note_attrs - expect(issue.notes[0].system_note_metadata.action).to eq 'cross_reference' - end + context 'with Issue' do + it_behaves_like 'import cross-referenced event' end - context 'when referenced in out of project issue/pull_request' do - it 'does not create expected note' do - importer.execute(issue_event) + context 'with MergeRequest' do + let(:issuable) { create(:merge_request, source_project: project, target_project: project) } - expect(issue.notes.count).to eq 0 - end + it_behaves_like 'import cross-referenced event' end end diff --git a/spec/lib/gitlab/github_import/importer/events/renamed_spec.rb b/spec/lib/gitlab/github_import/importer/events/renamed_spec.rb index 316ea798965..29598cb4354 100644 --- a/spec/lib/gitlab/github_import/importer/events/renamed_spec.rb +++ b/spec/lib/gitlab/github_import/importer/events/renamed_spec.rb @@ -8,7 +8,7 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::Renamed do let_it_be(:project) { create(:project, :repository) } let_it_be(:user) { create(:user) } - let(:issue) { create(:issue, project: project) } + let(:issuable) { create(:issue, project: project) } let(:client) { instance_double('Gitlab::GithubImport::Client') } let(:issue_event) do @@ -20,14 +20,14 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::Renamed do 'created_at' => '2022-04-26 18:30:53 UTC', 'old_title' => 'old title', 'new_title' => 'new title', - 'issue' => { 'number' => issue.iid } + 'issue' => { 'number' => issuable.iid, pull_request: issuable.is_a?(MergeRequest) } ) end let(:expected_note_attrs) do { - noteable_id: issue.id, - noteable_type: Issue.name, + noteable_id: issuable.id, + noteable_type: issuable.class.name, project_id: project.id, author_id: user.id, note: "changed title from **{-old-} title** to **{+new+} title**", @@ -48,31 +48,43 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::Renamed do describe '#execute' do before do allow_next_instance_of(Gitlab::GithubImport::IssuableFinder) do |finder| - allow(finder).to receive(:database_id).and_return(issue.id) + allow(finder).to receive(:database_id).and_return(issuable.id) end allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| allow(finder).to receive(:find).with(user.id, user.username).and_return(user.id) end end - it 'creates expected note' do - expect { importer.execute(issue_event) }.to change { issue.notes.count } - .from(0).to(1) - - expect(issue.notes.last) - .to have_attributes(expected_note_attrs) - end - - it 'creates expected system note metadata' do - expect { importer.execute(issue_event) }.to change { SystemNoteMetadata.count } + shared_examples 'import renamed event' do + it 'creates expected note' do + expect { importer.execute(issue_event) }.to change { issuable.notes.count } .from(0).to(1) - expect(SystemNoteMetadata.last) - .to have_attributes( - expected_system_note_metadata_attrs.merge( - note_id: Note.last.id + expect(issuable.notes.last) + .to have_attributes(expected_note_attrs) + end + + it 'creates expected system note metadata' do + expect { importer.execute(issue_event) }.to change { SystemNoteMetadata.count } + .from(0).to(1) + + expect(SystemNoteMetadata.last) + .to have_attributes( + expected_system_note_metadata_attrs.merge( + note_id: Note.last.id + ) ) - ) + end + end + + context 'with Issue' do + it_behaves_like 'import renamed event' + end + + context 'with MergeRequest' do + let(:issuable) { create(:merge_request, source_project: project, target_project: project) } + + it_behaves_like 'import renamed event' end end end diff --git a/spec/lib/gitlab/github_import/importer/events/reopened_spec.rb b/spec/lib/gitlab/github_import/importer/events/reopened_spec.rb index 2461dbb9701..354003fc997 100644 --- a/spec/lib/gitlab/github_import/importer/events/reopened_spec.rb +++ b/spec/lib/gitlab/github_import/importer/events/reopened_spec.rb @@ -9,7 +9,7 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::Reopened, :aggregate_fail let_it_be(:user) { create(:user) } let(:client) { instance_double('Gitlab::GithubImport::Client') } - let(:issue) { create(:issue, project: project) } + let(:issuable) { create(:issue, project: project) } let(:issue_event) do Gitlab::GithubImport::Representation::IssueEvent.from_json_hash( @@ -19,7 +19,7 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::Reopened, :aggregate_fail 'actor' => { 'id' => user.id, 'login' => user.username }, 'event' => 'reopened', 'created_at' => '2022-04-26 18:30:53 UTC', - 'issue' => { 'number' => issue.iid } + 'issue' => { 'number' => issuable.iid, pull_request: issuable.is_a?(MergeRequest) } ) end @@ -27,40 +27,61 @@ RSpec.describe Gitlab::GithubImport::Importer::Events::Reopened, :aggregate_fail { project_id: project.id, author_id: user.id, - target_id: issue.id, - target_type: Issue.name, + target_id: issuable.id, + target_type: issuable.class.name, action: 'reopened', created_at: issue_event.created_at, updated_at: issue_event.created_at }.stringify_keys end - let(:expected_state_event_attrs) do - { - user_id: user.id, - state: 'reopened', - created_at: issue_event.created_at - }.stringify_keys - end - before do allow_next_instance_of(Gitlab::GithubImport::IssuableFinder) do |finder| - allow(finder).to receive(:database_id).and_return(issue.id) + allow(finder).to receive(:database_id).and_return(issuable.id) end allow_next_instance_of(Gitlab::GithubImport::UserFinder) do |finder| allow(finder).to receive(:find).with(user.id, user.username).and_return(user.id) end end - it 'creates expected event and state event' do - importer.execute(issue_event) + shared_examples 'new event' do + it 'creates expected event and state event' do + importer.execute(issue_event) - expect(issue.events.count).to eq 1 - expect(issue.events[0].attributes) - .to include expected_event_attrs + expect(issuable.events.count).to eq 1 + expect(issuable.events[0].attributes) + .to include expected_event_attrs - expect(issue.resource_state_events.count).to eq 1 - expect(issue.resource_state_events[0].attributes) - .to include expected_state_event_attrs + expect(issuable.resource_state_events.count).to eq 1 + expect(issuable.resource_state_events[0].attributes) + .to include expected_state_event_attrs + end + end + + context 'with Issue' do + let(:expected_state_event_attrs) do + { + user_id: user.id, + issue_id: issuable.id, + state: 'reopened', + created_at: issue_event.created_at + }.stringify_keys + end + + it_behaves_like 'new event' + end + + context 'with MergeRequest' do + let(:issuable) { create(:merge_request, source_project: project, target_project: project) } + let(:expected_state_event_attrs) do + { + user_id: user.id, + merge_request_id: issuable.id, + state: 'reopened', + created_at: issue_event.created_at + }.stringify_keys + end + + it_behaves_like 'new event' end end diff --git a/spec/lib/gitlab/github_import/importer/issue_event_importer_spec.rb b/spec/lib/gitlab/github_import/importer/issue_event_importer_spec.rb index 33d5fbf13a0..5db49ee1a75 100644 --- a/spec/lib/gitlab/github_import/importer/issue_event_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/issue_event_importer_spec.rb @@ -42,10 +42,6 @@ RSpec.describe Gitlab::GithubImport::Importer::IssueEventImporter, :clean_gitlab end describe '#execute' do - before do - issue_event.attributes[:issue_db_id] = issue.id - end - context "when it's closed issue event" do let(:event_name) { 'closed' } diff --git a/spec/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer_spec.rb b/spec/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer_spec.rb index bb1ee79ad93..4ed01fd7e0b 100644 --- a/spec/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer_spec.rb @@ -6,7 +6,8 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointIssueEventsImporter let(:client) { double } let_it_be(:project) { create(:project, :import_started, import_source: 'http://somegithub.com') } - let_it_be(:issue) { create(:issue, project: project) } + + let!(:issuable) { create(:issue, project: project) } subject { described_class.new(project, client, parallel: parallel) } @@ -35,7 +36,7 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointIssueEventsImporter end describe '#page_counter_id' do - it { expect(subject.page_counter_id(issue)).to eq("issues/#{issue.iid}/issue_timeline") } + it { expect(subject.page_counter_id(issuable)).to eq("issues/#{issuable.iid}/issue_timeline") } end describe '#id_for_already_imported_cache' do @@ -51,6 +52,39 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointIssueEventsImporter end end + describe '#compose_associated_id!' do + let(:issuable) { build_stubbed(:issue, iid: 99) } + let(:event_resource) { Struct.new(:id, :event, :source, keyword_init: true) } + + context 'when event type is cross-referenced' do + let(:event) do + source_resource = Struct.new(:issue, keyword_init: true) + issue_resource = Struct.new(:id, keyword_init: true) + event_resource.new( + id: nil, + event: 'cross-referenced', + source: source_resource.new(issue: issue_resource.new(id: '100500')) + ) + end + + it 'assigns event id' do + subject.compose_associated_id!(issuable, event) + + expect(event.id).to eq 'cross-reference#99-in-100500' + end + end + + context "when event type isn't cross-referenced" do + let(:event) { event_resource.new(id: nil, event: 'labeled') } + + it "doesn't assign event id" do + subject.compose_associated_id!(issuable, event) + + expect(event.id).to eq nil + end + end + end + describe '#each_object_to_import', :clean_gitlab_redis_cache do let(:issue_event) do struct = Struct.new(:id, :event, :created_at, :issue, keyword_init: true) @@ -72,19 +106,37 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointIssueEventsImporter .with( :issue_timeline, project.import_source, - issue.iid, + issuable.iid, { state: 'all', sort: 'created', direction: 'asc', page: 1 } ).and_yield(page) end - it 'imports each issue event page by page' do - counter = 0 - subject.each_object_to_import do |object| - expect(object).to eq issue_event - expect(issue_event.issue['number']).to eq issue.iid - counter += 1 + context 'with issues' do + it 'imports each issue event page by page' do + counter = 0 + subject.each_object_to_import do |object| + expect(object).to eq issue_event + expect(issue_event.issue['number']).to eq issuable.iid + expect(issue_event.issue['pull_request']).to eq false + counter += 1 + end + expect(counter).to eq 1 + end + end + + context 'with merge requests' do + let!(:issuable) { create(:merge_request, source_project: project, target_project: project) } + + it 'imports each merge request event page by page' do + counter = 0 + subject.each_object_to_import do |object| + expect(object).to eq issue_event + expect(issue_event.issue['number']).to eq issuable.iid + expect(issue_event.issue['pull_request']).to eq true + counter += 1 + end + expect(counter).to eq 1 end - expect(counter).to eq 1 end it 'triggers page number increment' do @@ -103,7 +155,7 @@ RSpec.describe Gitlab::GithubImport::Importer::SingleEndpointIssueEventsImporter context 'when page is already processed' do before do page_counter = Gitlab::GithubImport::PageCounter.new( - project, subject.page_counter_id(issue) + project, subject.page_counter_id(issuable) ) page_counter.set(page.number) end diff --git a/spec/policies/issuable_policy_spec.rb b/spec/policies/issuable_policy_spec.rb index 706570babd5..fd7ec5917d6 100644 --- a/spec/policies/issuable_policy_spec.rb +++ b/spec/policies/issuable_policy_spec.rb @@ -18,8 +18,8 @@ RSpec.describe IssuablePolicy, models: true do project.add_reporter(reporter) end - def permissions(user, issue) - described_class.new(user, issue) + def permissions(user, issuable) + described_class.new(user, issuable) end describe '#rules' do @@ -153,5 +153,55 @@ RSpec.describe IssuablePolicy, models: true do expect(permissions(reporter, issue)).to be_allowed(:create_timelog) end end + + context 'when subject is a Merge Request' do + let(:issuable) { create(:merge_request) } + let(:policy) { permissions(user, issuable) } + + before do + allow(policy).to receive(:can?).with(:read_merge_request).and_return(can_read_merge_request) + end + + context 'when can_read_merge_request is false' do + let(:can_read_merge_request) { false } + + it 'does not allow :read_issuable' do + expect(policy).not_to be_allowed(:read_issuable) + end + end + + context 'when can_read_merge_request is true' do + let(:can_read_merge_request) { true } + + it 'allows :read_issuable' do + expect(policy).to be_allowed(:read_issuable) + end + end + end + + context 'when subject is an Issue' do + let(:issuable) { create(:issue) } + let(:policy) { permissions(user, issuable) } + + before do + allow(policy).to receive(:can?).with(:read_issue).and_return(can_read_issue) + end + + context 'when can_read_issue is false' do + let(:can_read_issue) { false } + + it 'does not allow :read_issuable' do + expect(policy).not_to be_allowed(:read_issuable) + end + end + + context 'when can_read_issue is true' do + let(:can_read_issue) { true } + + it 'allows :read_issuable' do + expect(policy).to be_allowed(:read_issuable) + end + end + end end end diff --git a/spec/requests/api/usage_data_queries_spec.rb b/spec/requests/api/usage_data_queries_spec.rb index 69a8d865a59..6ce03954246 100644 --- a/spec/requests/api/usage_data_queries_spec.rb +++ b/spec/requests/api/usage_data_queries_spec.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'spec_helper' +require 'rake_helper' RSpec.describe API::UsageDataQueries do include UsageDataHelpers @@ -64,5 +65,36 @@ RSpec.describe API::UsageDataQueries do expect(response).to have_gitlab_http_status(:forbidden) end end + + context 'when querying sql metrics' do + let(:file) { Rails.root.join('tmp', 'test', 'sql_metrics_queries.json') } + + before do + Rake.application.rake_require 'tasks/gitlab/usage_data' + + run_rake_task('gitlab:usage_data:generate_sql_metrics_queries') + end + + after do + FileUtils.rm_rf(file) + end + + it 'matches the generated query' do + Timecop.freeze(2021, 1, 1) do + get api(endpoint, admin) + end + + data = Gitlab::Json.parse(File.read(file)) + + expect( + json_response['counts_monthly'].except('aggregated_metrics') + ).to eq(data['counts_monthly'].except('aggregated_metrics')) + + expect(json_response['counts']).to eq(data['counts']) + expect(json_response['active_user_count']).to eq(data['active_user_count']) + expect(json_response['usage_activity_by_stage']).to eq(data['usage_activity_by_stage']) + expect(json_response['usage_activity_by_stage_monthly']).to eq(data['usage_activity_by_stage_monthly']) + end + end end end diff --git a/spec/services/issues/create_service_spec.rb b/spec/services/issues/create_service_spec.rb index 80c455e72b0..efff967614d 100644 --- a/spec/services/issues/create_service_spec.rb +++ b/spec/services/issues/create_service_spec.rb @@ -555,24 +555,29 @@ RSpec.describe Issues::CreateService do expect(reloaded_discussion.last_note.system).to eq(true) end - it 'assigns the title and description for the issue' do - issue = described_class.new(project: project, current_user: user, params: opts, spam_params: spam_params).execute + it 'sets default title and description values if not provided' do + issue = described_class.new( + project: project, current_user: user, + params: opts, + spam_params: spam_params + ).execute - expect(issue.title).not_to be_nil - expect(issue.description).not_to be_nil + expect(issue).to be_persisted + expect(issue.title).to eq("Follow-up from \"#{merge_request.title}\"") + expect(issue.description).to include("The following discussion from #{merge_request.to_reference} should be addressed") end - it 'can set nil explicitly to the title and description' do + it 'takes params from the request over the default values' do issue = described_class.new(project: project, current_user: user, - params: { - merge_request_to_resolve_discussions_of: merge_request, - description: nil, - title: nil - }, + params: opts.merge( + description: 'Custom issue description', + title: 'My new issue' + ), spam_params: spam_params).execute - expect(issue.description).to be_nil - expect(issue.title).to be_nil + expect(issue).to be_persisted + expect(issue.description).to eq('Custom issue description') + expect(issue.title).to eq('My new issue') end end @@ -594,24 +599,29 @@ RSpec.describe Issues::CreateService do expect(reloaded_discussion.last_note.system).to eq(true) end - it 'assigns the title and description for the issue' do - issue = described_class.new(project: project, current_user: user, params: opts, spam_params: spam_params).execute + it 'sets default title and description values if not provided' do + issue = described_class.new( + project: project, current_user: user, + params: opts, + spam_params: spam_params + ).execute - expect(issue.title).not_to be_nil - expect(issue.description).not_to be_nil + expect(issue).to be_persisted + expect(issue.title).to eq("Follow-up from \"#{merge_request.title}\"") + expect(issue.description).to include("The following discussion from #{merge_request.to_reference} should be addressed") end - it 'can set nil explicitly to the title and description' do + it 'takes params from the request over the default values' do issue = described_class.new(project: project, current_user: user, - params: { - merge_request_to_resolve_discussions_of: merge_request, - description: nil, - title: nil - }, + params: opts.merge( + description: 'Custom issue description', + title: 'My new issue' + ), spam_params: spam_params).execute - expect(issue.description).to be_nil - expect(issue.title).to be_nil + expect(issue).to be_persisted + expect(issue.description).to eq('Custom issue description') + expect(issue.title).to eq('My new issue') end end end diff --git a/spec/tasks/gitlab/usage_data_rake_spec.rb b/spec/tasks/gitlab/usage_data_rake_spec.rb index 442b884b313..f05b7876fdb 100644 --- a/spec/tasks/gitlab/usage_data_rake_spec.rb +++ b/spec/tasks/gitlab/usage_data_rake_spec.rb @@ -5,13 +5,20 @@ require 'rake_helper' RSpec.describe 'gitlab:usage data take tasks', :silence_stdout do include UsageDataHelpers + let(:metrics_file) { Rails.root.join('tmp', 'test', 'sql_metrics_queries.json') } + before do Rake.application.rake_require 'tasks/gitlab/usage_data' + # stub prometheus external http calls https://gitlab.com/gitlab-org/gitlab/-/issues/245277 stub_prometheus_queries stub_database_flavor_check end + after do + FileUtils.rm_rf(metrics_file) + end + describe 'dump_sql_in_yaml' do it 'dumps SQL queries in yaml format' do expect { run_rake_task('gitlab:usage_data:dump_sql_in_yaml') }.to output(/.*recorded_at:.*/).to_stdout @@ -23,4 +30,12 @@ RSpec.describe 'gitlab:usage data take tasks', :silence_stdout do expect { run_rake_task('gitlab:usage_data:dump_sql_in_json') }.to output(/.*"recorded_at":.*/).to_stdout end end + + describe 'generate_sql_metrics_fixture' do + it 'generates fixture file correctly' do + run_rake_task('gitlab:usage_data:generate_sql_metrics_queries') + + expect(Pathname.new(metrics_file)).to exist + end + end end diff --git a/tooling/config/CODEOWNERS.yml b/tooling/config/CODEOWNERS.yml index 71818b67ab1..6b24134ea17 100644 --- a/tooling/config/CODEOWNERS.yml +++ b/tooling/config/CODEOWNERS.yml @@ -47,7 +47,7 @@ - 'jira_connect/' - 'kubernetes/' - 'protected_environments/' - - '/config/feature_flags/development/jira_connect_*' + - '/config/feature_flags/**/*' - '/config/metrics/' - '/app/controllers/groups/dependency_proxy_auth_controller.rb' - '/app/finders/ci/auth_job_finder.rb' @@ -65,17 +65,19 @@ keywords: - audit patterns: - - '**%{keyword}**' + - '/{,ee/}app/**/*%{keyword}*' + - '/{,ee/}config/**/*%{keyword}*' + - '/{,ee/}lib/**/*%{keyword}*' deny: keywords: - '*.png' - '*bundler-audit*' - '**/merge_requests/**' - - '/ee/app/services/audit_events/*' + - '/config/feature_flags/**/*' + - '/ee/app/services/audit_events/**/*' - '/ee/config/feature_flags/development/auditor_group_runner_access.yml' - - '/ee/spec/services/audit_events/*' + - '/ee/spec/services/audit_events/**/*' - '/ee/spec/services/ci/*' - '/ee/spec/services/personal_access_tokens/*' - - '/qa/**/*' patterns: - '%{keyword}'