From b2a110621cd15508941d15a2482dd18d0cb8836a Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 29 Oct 2024 03:30:57 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .../projects/compare/rapid_diffs/index.js | 3 + .../page_bundles/notes/_timeline.scss | 17 +- .../themes/dark_mode_overrides.scss | 6 - .../projects/compare_controller.rb | 16 +- app/models/project_import_data.rb | 4 + .../projects/compare/rapid_diffs.html.haml | 34 ++ .../feature_flags/wip/gitea_user_mapping.yml | 9 + config/routes/repository.rb | 5 + doc/development/migration_style_guide.md | 20 +- .../merge_request_approval_policies.md | 2 +- doc/user/project/import/gitea.md | 9 + doc/user/project/import/index.md | 10 + .../legacy_github_import/base_formatter.rb | 45 +- .../legacy_github_import/comment_formatter.rb | 18 +- lib/gitlab/legacy_github_import/importer.rb | 76 ++- .../issuable_formatter.rb | 24 +- .../legacy_github_import/issue_formatter.rb | 16 + .../legacy_github_import/label_formatter.rb | 6 +- .../milestone_formatter.rb | 4 + .../legacy_github_import/project_creator.rb | 16 +- .../pull_request_formatter.rb | 16 + .../legacy_github_import/user_formatter.rb | 51 +- locale/gitlab.pot | 7 +- .../projects/compare_controller_spec.rb | 33 ++ spec/factories/projects.rb | 6 + .../fix_pick_up_at_ci_deleted_object_spec.rb | 2 +- .../base_formatter_spec.rb | 8 +- .../comment_formatter_spec.rb | 220 ++++++- .../legacy_github_import/importer_spec.rb | 543 ++++++++++-------- .../issuable_formatter_spec.rb | 8 + .../issue_formatter_spec.rb | 283 ++++++++- .../label_formatter_spec.rb | 48 +- .../milestone_formatter_spec.rb | 155 +++-- .../project_creator_spec.rb | 42 +- .../pull_request_formatter_spec.rb | 281 ++++++++- .../release_formatter_spec.rb | 3 +- .../user_formatter_spec.rb | 185 ++++-- spec/models/project_import_data_spec.rb | 20 + workhorse/go.mod | 10 +- workhorse/go.sum | 20 +- 40 files changed, 1779 insertions(+), 502 deletions(-) create mode 100644 app/assets/javascripts/pages/projects/compare/rapid_diffs/index.js create mode 100644 app/views/projects/compare/rapid_diffs.html.haml create mode 100644 config/feature_flags/wip/gitea_user_mapping.yml diff --git a/app/assets/javascripts/pages/projects/compare/rapid_diffs/index.js b/app/assets/javascripts/pages/projects/compare/rapid_diffs/index.js new file mode 100644 index 00000000000..b86c9ec442f --- /dev/null +++ b/app/assets/javascripts/pages/projects/compare/rapid_diffs/index.js @@ -0,0 +1,3 @@ +import initCompareSelector from '~/projects/compare'; + +initCompareSelector(); diff --git a/app/assets/stylesheets/page_bundles/notes/_timeline.scss b/app/assets/stylesheets/page_bundles/notes/_timeline.scss index cb840a6b86e..68003878844 100644 --- a/app/assets/stylesheets/page_bundles/notes/_timeline.scss +++ b/app/assets/stylesheets/page_bundles/notes/_timeline.scss @@ -1,3 +1,14 @@ +:root { + --timeline-entry-internal-note-background-color: var(--gl-color-orange-50); + --timeline-entry-target-background-color : var(--gl-color-blue-50); +} + +// stylelint-disable-next-line gitlab/no-gl-class +:root.gl-dark { + --timeline-entry-internal-note-background-color: #453522; + --timeline-entry-target-background-color : #2A394E; +} + .timeline { margin: 0; padding: 0; @@ -29,7 +40,7 @@ @apply gl-text-default; &:not(.note-form).internal-note .timeline-content { - @apply gl-bg-orange-50 #{!important}; + background-color: var(--timeline-entry-internal-note-background-color) !important; } .timeline-entry-inner { @@ -39,7 +50,7 @@ &:target, &.target { .timeline-content { - @apply gl-bg-blue-50 #{!important}; + background-color: var(--timeline-entry-target-background-color) !important; } + .discussion-reply-holder { @@ -47,7 +58,7 @@ } &.system-note .note-body .note-text.system-note-commit-list::after { - background: linear-gradient(rgba(var(--blue-50), 0.1) -100px, var(--blue-50) 100%); + background: linear-gradient(rgba(var(--timeline-entry-target-background-color), 0.1) -100px, var(--timeline-entry-target-background-color) 100%); } } diff --git a/app/assets/stylesheets/themes/dark_mode_overrides.scss b/app/assets/stylesheets/themes/dark_mode_overrides.scss index 9c2a0730e87..e2083671e77 100644 --- a/app/assets/stylesheets/themes/dark_mode_overrides.scss +++ b/app/assets/stylesheets/themes/dark_mode_overrides.scss @@ -41,9 +41,3 @@ background-color: $gray-200; } } - -.timeline-entry.internal-note:not(.note-form) .timeline-content, -.timeline-entry.draft-note:not(.note-form) .timeline-content { - // soften on darkmode - background-color: mix($gray-50, $orange-50, 75%) !important; -} diff --git a/app/controllers/projects/compare_controller.rb b/app/controllers/projects/compare_controller.rb index 673dc1838d6..ac859b96b8e 100644 --- a/app/controllers/projects/compare_controller.rb +++ b/app/controllers/projects/compare_controller.rb @@ -12,11 +12,11 @@ class Projects::CompareController < Projects::ApplicationController before_action :require_non_empty_project before_action :authorize_read_code! # Defining ivars - before_action :define_diffs, only: [:show, :diff_for_path] - before_action :define_environment, only: [:show] - before_action :define_diff_notes_disabled, only: [:show, :diff_for_path] - before_action :define_commits, only: [:show, :diff_for_path, :signatures] - before_action :merge_request, only: [:index, :show] + before_action :define_diffs, only: [:show, :diff_for_path, :rapid_diffs] + before_action :define_environment, only: [:show, :rapid_diffs] + before_action :define_diff_notes_disabled, only: [:show, :diff_for_path, :rapid_diffs] + before_action :define_commits, only: [:show, :diff_for_path, :signatures, :rapid_diffs] + before_action :merge_request, only: [:index, :show, :rapid_diffs] # Validation before_action :validate_refs! @@ -72,6 +72,12 @@ class Projects::CompareController < Projects::ApplicationController end end + def rapid_diffs + return render_404 unless ::Feature.enabled?(:rapid_diffs, current_user, type: :wip) + + show + end + private def build_from_to_vars diff --git a/app/models/project_import_data.rb b/app/models/project_import_data.rb index 96c1ad7def8..30cdb6ea62d 100644 --- a/app/models/project_import_data.rb +++ b/app/models/project_import_data.rb @@ -46,4 +46,8 @@ class ProjectImportData < ApplicationRecord def clear_credentials self.credentials = {} end + + def user_mapping_enabled? + self.data&.dig('user_contribution_mapping_enabled') || false + end end diff --git a/app/views/projects/compare/rapid_diffs.html.haml b/app/views/projects/compare/rapid_diffs.html.haml new file mode 100644 index 00000000000..0db9469a19d --- /dev/null +++ b/app/views/projects/compare/rapid_diffs.html.haml @@ -0,0 +1,34 @@ +- @no_container = true +- container_class = fluid_layout ? '' : 'container-limited' +- add_to_breadcrumbs s_("CompareRevisions|Compare revisions"), project_compare_index_path(@project) +- page_title "#{params[:from]} to #{params[:to]}" +- has_diff = @commits.present? || @diffs.present? && @diffs.diff_files.present? +-# Only show commit list in the first page +- hide_commit_list = params[:page].present? && params[:page] != '1' + +.container-fluid{ class: [container_class] } + .gl-border-b-0.gl-mb-0.gl-pt-4 + .js-signature-container{ data: { 'signatures-path' => signatures_namespace_project_compare_index_path } } + #js-compare-selector{ data: project_compare_selector_data(@project, @merge_request, params) } + +- if has_diff + .container-fluid{ class: [container_class] } + = render "projects/commits/commit_list" unless hide_commit_list + .container-fluid + = render RapidDiffs::DiffFileComponent.with_collection(@diffs.diff_files, parallel_view: diff_view == :parallel) +- else + .container-fluid + = render Pajamas::CardComponent.new(card_options: { class: "gl-bg-gray-10" }) do |c| + - c.with_body do + = render Pajamas::EmptyStateComponent.new(svg_path: 'illustrations/empty-state/empty-commit-md.svg', + title: s_("CompareRevisions|There isn't anything to compare")) do |c| + + - c.with_description do + - if params[:to] == params[:from] + - source_branch = capture do + %span.ref-name= params[:from] + - target_branch = capture do + %span.ref-name= params[:to] + = (s_("CompareRevisions|%{source_branch} and %{target_branch} are the same.") % { source_branch: source_branch, target_branch: target_branch }).html_safe + - else + = _("To get a valid comparison, select two different branches.") diff --git a/config/feature_flags/wip/gitea_user_mapping.yml b/config/feature_flags/wip/gitea_user_mapping.yml new file mode 100644 index 00000000000..9743bf2ada8 --- /dev/null +++ b/config/feature_flags/wip/gitea_user_mapping.yml @@ -0,0 +1,9 @@ +--- +name: gitea_user_mapping +feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/467084 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/167078 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/498390 +milestone: '17.6' +group: group::import and integrate +type: wip +default_enabled: false diff --git a/config/routes/repository.rb b/config/routes/repository.rb index 6b03528990e..2efea11c268 100644 --- a/config/routes/repository.rb +++ b/config/routes/repository.rb @@ -6,6 +6,11 @@ # Don't use format parameter as file extension (old 3.0.x behavior) # See http://guides.rubyonrails.org/routing.html#route-globbing-and-wildcard-segments scope format: false do + get '/compare/:from...:to/', to: 'compare#rapid_diffs', as: 'rapid_diffs_compare', + constraints: ->(params) { params[:rapid_diffs] == 'true' && params[:from] =~ /.+/ && params[:to] =~ /.+/ } + get '/compare/:from..:to/', to: 'compare#rapid_diffs', as: 'rapid_diffs_compare_with_two_dots', + constraints: ->(params) { params[:rapid_diffs] == 'true' && params[:from] =~ /.+/ && params[:to] =~ /.+/ }, defaults: { straight: "true" } + get '/compare/:from...:to/', to: 'compare#show', as: 'compare', constraints: { from: /.+/, to: /.+/ } get '/compare/:from..:to/', to: 'compare#show', as: 'compare_with_two_dots', constraints: { from: /.+/, to: /.+/ }, defaults: { straight: "true" } diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index 679de383277..c4d0eef6fca 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -196,14 +196,28 @@ VERSION= bundle exec rails db:migrate:main After a table has been created, it should be added to the database dictionary, following the steps mentioned in the [database dictionary guide](database/database_dictionary.md#adding-tables). -### Migration checksum +### Migration checksum file -When a migration is first executed, a new file is created in [db/schema_migrations](https://gitlab.com/gitlab-org/gitlab/-/tree/v17.5.0-ee/db/schema_migrations) containing a `SHA` generated from the migration's timestamp. The name of this new file is the same as the [timestamp portion](#migration-timestamp-age) of the migration filename, for example [db/schema_migrations/20241021120146](https://gitlab.com/gitlab-org/gitlab/blob/aa7cfb42c312/db/schema_migrations/20241021120146). +When a migration is first executed, a new `migration checksum file` is created in [db/schema_migrations](https://gitlab.com/gitlab-org/gitlab/-/tree/v17.5.0-ee/db/schema_migrations) containing a `SHA256` generated from the migration's timestamp. The name of this new file is the same as the [timestamp portion](#migration-timestamp-age) of the migration filename, for example [db/schema_migrations/20241021120146](https://gitlab.com/gitlab-org/gitlab/blob/aa7cfb42c312/db/schema_migrations/20241021120146). The content of this file is the `SHA256` of the timestamp portion, for example: -This new `db/schema_migration/` file indicates that the migration was executed successfuly and the result recorded in `db/structure.sql`. The presence of this file prevents the same migration from being executed twice, and therefore, it's necessary to include this file in the merge request that adds the new migration. +```shell +$ echo -n "20241021120146" | sha256sum +7a3e382a6e5564bfa7004bca1a357a910b151e7399c6466113daf01526d97470 - +``` + +The `SHA256` adds unique content to the file so Git rename detection sees them as [separate files](https://gitlab.com/gitlab-org/gitlab/-/issues/218590#note_384712827). + +This `migration checksum file` indicates that the migration executed successfuly and the result recorded in `db/structure.sql`. The presence of this file prevents the same migration from being executed twice, and therefore, it's necessary to include this file in the merge request that adds the new migration. See [Development change: Database schema version handling outside of structure.sql](https://gitlab.com/gitlab-org/gitlab/-/issues/218590) for more details about the `db/schema_migrations` directory. +#### Keeping the migration checksum file up-to-date + +- when a new migration is created, run `rake db:migrate` to execute the migration and generate the corresponding `db/schema_migration/` checksum file, and add this file into version control. +- if the migration is deleted, remove the corresponding `db/schema_migration/` checksum file. +- if the _timestamp portion_ of the migration is changed, remove the corresponding `db/schema_migration/` checksum file and run `rake db:migrate` to generate a new one, and add this file into version control. +- if the content of the migration is changed, no changes are required to the `db/schema_migration/` checksum file. + ## Avoiding downtime The document ["Avoiding downtime in migrations"](database/avoiding_downtime_in_migrations.md) specifies diff --git a/doc/user/application_security/policies/merge_request_approval_policies.md b/doc/user/application_security/policies/merge_request_approval_policies.md index 4142407c765..54d70a4a456 100644 --- a/doc/user/application_security/policies/merge_request_approval_policies.md +++ b/doc/user/application_security/policies/merge_request_approval_policies.md @@ -259,7 +259,7 @@ The settings set in the policy overwrite settings in the project. | Field | Type | Required | Possible values | Applicable rule type | Description | |-------------------------------------|-----------------------|----------|---------------------------------------------------------------|----------------------|-------------| | `block_branch_modification` | `boolean` | false | `true`, `false` | All | When enabled, prevents a user from removing a branch from the protected branches list, deleting a protected branch, or changing the default branch if that branch is included in the security policy. This ensures users cannot remove protection status from a branch to merge vulnerable code. Enforced based on `branches`, `branch_type` and `policy_scope` and regardless of detected vulnerabilities. | -| `block_group_branch_modification` | `boolean` or `object` | false | `true`, `false`, `{ enabled: boolean, exceptions: [string] }` | All | When enabled, prevents a user from removing group-level protected branches on every group the policy applies to. If `block_branch_modification` is `true`, implicitly defaults to `true`. Enforced based on `branches`, `branch_type` and `policy_scope` and regardless of detected vulnerabilities. | +| `block_group_branch_modification` | `boolean` or `object` | false | `true`, `false`, `{ enabled: boolean, exceptions: [string] }` | All | When enabled, prevents a user from removing group-level protected branches on every group the policy applies to. If `block_branch_modification` is `true`, implicitly defaults to `true`. Enforced based on `branches`, `branch_type` and `policy_scope` and regardless of detected vulnerabilities. Add top-level groups that support [group-level protected branches](../../../user/project/repository/branches/protected.md#for-all-projects-in-a-group) as `exceptions` | | `prevent_approval_by_author` | `boolean` | false | `true`, `false` | `Any merge request` | When enabled, merge request authors cannot approve their own MRs. This ensures code authors cannot introduce vulnerabilities and approve code to merge. | | `prevent_approval_by_commit_author` | `boolean` | false | `true`, `false` | `Any merge request` | When enabled, users who have contributed code to the MR are ineligible for approval. This ensures code committers cannot introduce vulnerabilities and approve code to merge. | | `remove_approvals_with_new_commit` | `boolean` | false | `true`, `false` | `Any merge request` | When enabled, if an MR receives all necessary approvals to merge, but then a new commit is added, new approvals are required. This ensures new commits that may include vulnerabilities cannot be introduced. | diff --git a/doc/user/project/import/gitea.md b/doc/user/project/import/gitea.md index 41cb86c7abf..c3cf2aed44d 100644 --- a/doc/user/project/import/gitea.md +++ b/doc/user/project/import/gitea.md @@ -92,3 +92,12 @@ You also can: - Filter projects by name. If a filter is applied, **Import all projects** imports only selected projects. - Choose a different name for the project and a different namespace if you have the privileges to do so. + +## User contribution mapping + +User contributions are assigned to the project creator (usually the user who started the import process) by default. +This method of user contribution mapping is available for GitLab self-managed without enabled feature flags. + +For information on the other method available for GitLab self-managed +with enabled feature flags and for GitLab.com, +see [User contribution and membership mapping](../../project/import/index.md#user-contribution-and-membership-mapping). diff --git a/doc/user/project/import/index.md b/doc/user/project/import/index.md index 42f4ceca3db..edb4a5709ef 100644 --- a/doc/user/project/import/index.md +++ b/doc/user/project/import/index.md @@ -86,6 +86,7 @@ DETAILS: **Offering:** GitLab.com, Self-managed > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/443557) to direct transfer migrations for self-managed instances in GitLab 17.4 [with flags](../../../administration/feature_flags.md) named `importer_user_mapping` and `bulk_import_importer_user_mapping`. Disabled by default. +> - [Introduced to Gitea project import](https://gitlab.com/gitlab-org/gitlab/-/issues/467084) in GitLab 17.6 [with flags](../../../administration/feature_flags.md) named `importer_user_mapping` and `gitea_user_mapping`. Disabled by default. FLAG: The availability of this feature is controlled by feature flags. @@ -129,6 +130,15 @@ to existing users on the destination instance. Until they are reassigned, contributions display as associated with the placeholder. Placeholder memberships do not display in member lists. +#### Exceptions + +A placeholder user is created for each user on the source instance, except in the following scenarios: + +- You are importing a project from [Gitea](gitea.md) and the user has been deleted on Gitea before the import. + Contributions from these "ghost users" are mapped to the user who imported the project and not to a placeholder user. +- You have exceeded your [placeholder user limit](#placeholder-user-limits). Contributions from any new users after exceeding your limit are + mapped to a single import user. + #### Placeholder user attributes Placeholder users are different to regular users and cannot: diff --git a/lib/gitlab/legacy_github_import/base_formatter.rb b/lib/gitlab/legacy_github_import/base_formatter.rb index 9bbf68cd9c9..9e6e08c37a3 100644 --- a/lib/gitlab/legacy_github_import/base_formatter.rb +++ b/lib/gitlab/legacy_github_import/base_formatter.rb @@ -3,17 +3,25 @@ module Gitlab module LegacyGithubImport class BaseFormatter - attr_reader :client, :formatter, :project, :raw_data + attr_reader :client, :formatter, :project, :raw_data, :source_user_mapper - def initialize(project, raw_data, client = nil) + def initialize(project, raw_data, client = nil, source_user_mapper = nil) @project = project @raw_data = raw_data @client = client @formatter = Gitlab::ImportFormatter.new + @source_user_mapper = source_user_mapper end - # rubocop: disable CodeReuse/ActiveRecord def create! + record = create_record + push_placeholder_references(record) + + record + end + + # rubocop: disable CodeReuse/ActiveRecord -- Existing legacy code + def create_record association = project.public_send(project_association) # rubocop:disable GitlabSecurity/PublicSend association.find_or_create_by!(find_condition) do |record| @@ -22,6 +30,31 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + def push_placeholder_references(record, contributing_users: nil) + contributing_users ||= contributing_user_formatters + + contributing_users.each do |user_reference_column, user_formatter| + push_placeholder_reference(record, user_reference_column, user_formatter.source_user) + end + end + + def push_placeholder_reference(record, user_reference_column, source_user) + return unless project.import_data.user_mapping_enabled? + + user_id = record[user_reference_column] + + return if user_id.nil? || source_user.nil? + return if source_user.accepted_status? && user_id == source_user.reassign_to_user_id + + ::Import::PlaceholderReferences::PushService.from_record( + import_source: imported_from, + import_uid: project.import_state.id, + record: record, + source_user: source_user, + user_reference_column: user_reference_column + ).execute + end + def url raw_data[:url] || '' end @@ -32,6 +65,12 @@ module Gitlab ::Import::SOURCE_NONE end + + # A hash of user_reference_columns and its corresponding UserFormatter objects must be defined on each formatter + # in order to save it using #create! + def contributing_user_formatters + raise NotImplementedError + end end end end diff --git a/lib/gitlab/legacy_github_import/comment_formatter.rb b/lib/gitlab/legacy_github_import/comment_formatter.rb index acf1729ff09..2086c3eac3d 100644 --- a/lib/gitlab/legacy_github_import/comment_formatter.rb +++ b/lib/gitlab/legacy_github_import/comment_formatter.rb @@ -5,6 +5,8 @@ module Gitlab class CommentFormatter < BaseFormatter attr_writer :author_id + attr_accessor :gitlab_issuable + def attributes { project: project, @@ -19,10 +21,24 @@ module Gitlab } end + def project_association + :notes + end + + def create_record + gitlab_issuable.notes.create!(attributes) + end + + def contributing_user_formatters + { + author_id: author + } + end + private def author - @author ||= UserFormatter.new(client, raw_data[:user]) + @author ||= UserFormatter.new(client, raw_data[:user], project, source_user_mapper) end def author_id diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 94522de8416..51414c3d09e 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -3,6 +3,11 @@ module Gitlab module LegacyGithubImport class Importer + include Gitlab::Utils::StrongMemoize + + PLACEHOLDER_LOAD_SLEEP = 3 + PLACEHOLDER_LOAD_TIMEOUT = 300 + def self.refmap Gitlab::GithubImport.refmap end @@ -18,8 +23,6 @@ module Gitlab end def client - return @client if defined?(@client) - unless credentials raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" @@ -38,6 +41,16 @@ module Gitlab @client = Client.new(credentials[:user], **opts) end + strong_memoize_attr :client + + def source_user_mapper + Gitlab::Import::SourceUserMapper.new( + namespace: project.root_ancestor, + import_type: project.import_type, + source_hostname: client.host + ) + end + strong_memoize_attr :source_user_mapper def execute # The ordering of importing is important here due to the way GitHub structures their data @@ -67,6 +80,7 @@ module Gitlab # 2) https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89694/diffs#dfc4a8141aa296465ea3c50b095a30292fb6ebc4_180_182 import_releases unless project.gitea_import? + wait_for_placeholder_references handle_errors true @@ -118,7 +132,7 @@ module Gitlab fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| issues.each do |raw| raw = raw.to_h - gh_issue = IssueFormatter.new(project, raw, client) + gh_issue = IssueFormatter.new(project, raw, client, source_user_mapper) begin issuable = @@ -134,6 +148,8 @@ module Gitlab end end end + + load_placeholder_references end # rubocop: enable CodeReuse/ActiveRecord @@ -141,7 +157,7 @@ module Gitlab fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |prs| prs.each do |raw| raw = raw.to_h - gh_pull_request = PullRequestFormatter.new(project, raw, client) + gh_pull_request = PullRequestFormatter.new(project, raw, client, source_user_mapper) next unless gh_pull_request.valid? @@ -166,6 +182,7 @@ module Gitlab end project.repository.after_remove_branch + load_placeholder_references end def restore_source_branch(pull_request) @@ -223,6 +240,8 @@ module Gitlab create_comments(comments) end + + load_placeholder_references end # rubocop: enable CodeReuse/ActiveRecord @@ -232,7 +251,7 @@ module Gitlab comments.each do |raw| raw = raw.to_h - comment = CommentFormatter.new(project, raw, client) + comment = CommentFormatter.new(project, raw, client, source_user_mapper) # GH does not return info about comment's parent, so we guess it by checking its URL! *_, parent, iid = URI(raw[:html_url]).path.split('/') @@ -245,7 +264,8 @@ module Gitlab next unless issuable - issuable.notes.create!(comment.attributes) + comment.gitlab_issuable = issuable + comment.create! rescue StandardError => e errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw[:html_url]), errors: e.message } end @@ -316,6 +336,50 @@ module Gitlab errors << { type: resource_type, errors: e.message } end + def load_placeholder_references + return unless project.import_data.user_mapping_enabled? + + ::Import::LoadPlaceholderReferencesWorker.perform_async( + project.import_type, + project.import_state.id, + 'current_user_id' => project.creator_id + ) + end + + def placeholder_references_loaded? + return true unless project.import_data.user_mapping_enabled? + + ::Import::PlaceholderReferences::Store.new( + import_source: project.import_type, + import_uid: project.import_state.id + ).empty? + end + + def wait_for_placeholder_references + # Since this importer is synchronous, wait until all placeholder references have been saved + # to the database before completing the import + time_waited = 0 + + until time_waited >= PLACEHOLDER_LOAD_TIMEOUT || placeholder_references_loaded? + Kernel.sleep PLACEHOLDER_LOAD_SLEEP + time_waited += PLACEHOLDER_LOAD_SLEEP + end + + if placeholder_references_loaded? + return if time_waited == 0 + + ::Import::Framework::Logger.info( + message: "Placeholder references finished loading to database after #{time_waited} seconds.", + import_source: project.import_type, + import_uid: project.import_state.id + ) + else + timeout_error = "Timed out after waiting #{PLACEHOLDER_LOAD_TIMEOUT} seconds " \ + "for placeholder references to finish saving" + errors << { type: :placeholder_references, errors: timeout_error } + end + end + def imported?(resource_type) Rails.cache.read("#{cache_key_prefix}:#{resource_type}:imported") end diff --git a/lib/gitlab/legacy_github_import/issuable_formatter.rb b/lib/gitlab/legacy_github_import/issuable_formatter.rb index e4e333735be..a1a0be8d343 100644 --- a/lib/gitlab/legacy_github_import/issuable_formatter.rb +++ b/lib/gitlab/legacy_github_import/issuable_formatter.rb @@ -17,6 +17,26 @@ module Gitlab { iid: number } end + def create! + record = super + + return record unless assignee_id + + # Fetch first assignee because Gitea's API only returns one assignee for issue assignees + assignee_record = record.method(project_assignee_association).call.first + push_placeholder_references(assignee_record, contributing_users: contributing_assignee_formatters) + + record + end + + def project_assignee_association + raise NotImplementedError + end + + def contributing_assignee_formatters + raise NotImplementedError + end + private def state @@ -28,7 +48,7 @@ module Gitlab end def author - @author ||= UserFormatter.new(client, raw_data[:user]) + @author ||= UserFormatter.new(client, raw_data[:user], project, source_user_mapper) end def author_id @@ -37,7 +57,7 @@ module Gitlab def assignee if assigned? - @assignee ||= UserFormatter.new(client, raw_data[:assignee]) + @assignee ||= UserFormatter.new(client, raw_data[:assignee], project, source_user_mapper) end end diff --git a/lib/gitlab/legacy_github_import/issue_formatter.rb b/lib/gitlab/legacy_github_import/issue_formatter.rb index 8b94307337e..a50f19c0c25 100644 --- a/lib/gitlab/legacy_github_import/issue_formatter.rb +++ b/lib/gitlab/legacy_github_import/issue_formatter.rb @@ -30,6 +30,22 @@ module Gitlab def pull_request? raw_data[:pull_request].present? end + + def project_assignee_association + :issue_assignees + end + + def contributing_user_formatters + { + author_id: author + } + end + + def contributing_assignee_formatters + { + user_id: assignee + } + end end end end diff --git a/lib/gitlab/legacy_github_import/label_formatter.rb b/lib/gitlab/legacy_github_import/label_formatter.rb index e3b767f41fa..953c87aa56c 100644 --- a/lib/gitlab/legacy_github_import/label_formatter.rb +++ b/lib/gitlab/legacy_github_import/label_formatter.rb @@ -15,7 +15,7 @@ module Gitlab :labels end - def create! + def create_record params = attributes.except(:project) service = ::Labels::FindOrCreateService.new(nil, project, params) label = service.execute(skip_authorization: true) @@ -25,6 +25,10 @@ module Gitlab label end + def contributing_user_formatters + {} + end + private def color diff --git a/lib/gitlab/legacy_github_import/milestone_formatter.rb b/lib/gitlab/legacy_github_import/milestone_formatter.rb index 60d5bcbf44a..90e56b58148 100644 --- a/lib/gitlab/legacy_github_import/milestone_formatter.rb +++ b/lib/gitlab/legacy_github_import/milestone_formatter.rb @@ -32,6 +32,10 @@ module Gitlab end end + def contributing_user_formatters + {} + end + private def state diff --git a/lib/gitlab/legacy_github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb index 53c4d767ce3..643ac6edcf6 100644 --- a/lib/gitlab/legacy_github_import/project_creator.rb +++ b/lib/gitlab/legacy_github_import/project_creator.rb @@ -5,7 +5,7 @@ module Gitlab class ProjectCreator attr_reader :repo, :name, :namespace, :current_user, :session_data, :type - def initialize(repo, name, namespace, current_user, type: 'github', **session_data) + def initialize(repo, name, namespace, current_user, type: :github, **session_data) @repo = repo @name = name @namespace = namespace @@ -25,7 +25,12 @@ module Gitlab import_type: type, import_source: repo[:full_name], import_url: import_url, - skip_wiki: skip_wiki + skip_wiki: skip_wiki, + import_data: { + data: { + user_contribution_mapping_enabled: user_contribution_mapping_enabled + } + } }.merge!(extra_attrs) ::Projects::CreateService.new(current_user, attrs).execute @@ -52,6 +57,13 @@ module Gitlab def skip_wiki repo[:has_wiki] end + + # This checks if user mapping is enabled for Gitea only since GitHub UCM is not yet implemented + def user_contribution_mapping_enabled + return false if type != ::Import::SOURCE_GITEA + + Feature.enabled?(:importer_user_mapping, current_user) && Feature.enabled?(:gitea_user_mapping, current_user) + end end end end diff --git a/lib/gitlab/legacy_github_import/pull_request_formatter.rb b/lib/gitlab/legacy_github_import/pull_request_formatter.rb index 9afd859b995..138a23df7fd 100644 --- a/lib/gitlab/legacy_github_import/pull_request_formatter.rb +++ b/lib/gitlab/legacy_github_import/pull_request_formatter.rb @@ -78,6 +78,22 @@ module Gitlab state == 'opened' end + def project_assignee_association + :merge_request_assignees + end + + def contributing_user_formatters + { + author_id: author + } + end + + def contributing_assignee_formatters + { + user_id: assignee + } + end + private def state diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index a57edcc7ba4..2f4d107b8e0 100644 --- a/lib/gitlab/legacy_github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -3,13 +3,17 @@ module Gitlab module LegacyGithubImport class UserFormatter - attr_reader :client, :raw + include Gitlab::Utils::StrongMemoize + + attr_reader :client, :raw, :project, :source_user_mapper GITEA_GHOST_EMAIL = 'ghost_user@gitea_import_dummy_email.com' - def initialize(client, raw) + def initialize(client, raw, project, source_user_mapper) @client = client @raw = raw + @project = project + @source_user_mapper = source_user_mapper end def id @@ -21,30 +25,59 @@ module Gitlab end def gitlab_id - return @gitlab_id if defined?(@gitlab_id) - - @gitlab_id = find_by_email + project.import_data.user_mapping_enabled? ? gitlab_user&.id : find_by_email end + strong_memoize_attr :gitlab_id + + def source_user + return if !project.import_data.user_mapping_enabled? || ghost_user? + + source_user_mapper.find_or_create_source_user( + source_name: gitea_user[:login], + source_username: gitea_user[:full_name] || gitea_user[:login], + source_user_identifier: raw[:id] + ) + end + strong_memoize_attr :source_user private - def email + def ghost_user? + raw[:login] == 'Ghost' && raw[:id] == -1 + end + + def gitea_user # Gitea marks deleted users as 'Ghost' users and removes them from # their system. So for Gitea 'Ghost' users we need to assign a dummy # email address to avoid querying the Gitea api for a non existing user - if raw[:login] == 'Ghost' && raw[:id] == -1 - @email = GITEA_GHOST_EMAIL + user_hash = {} + + if ghost_user? + user_hash[:login] = user_hash[:full_name] = raw[:login] + user_hash[:email] = GITEA_GHOST_EMAIL else - @email ||= client.user(raw[:login]).to_h[:email] + user_hash = client.user(raw[:login]).to_h.slice(:id, :login, :full_name, :email) end + + user_hash end + strong_memoize_attr :gitea_user def find_by_email + email = gitea_user[:email] + return unless email User.find_by_any_email(email) .try(:id) end + + def gitlab_user + return if ghost_user? + + source_user.mapped_user + end + strong_memoize_attr :gitlab_user end end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 8f97b273f33..8d61236f061 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -48345,7 +48345,7 @@ msgstr "" msgid "ScanResultPolicy|Prevent %{linkStart}group branch%{linkEnd} modification %{exceptSelection}" msgstr "" -msgid "ScanResultPolicy|Prevent %{linkStart}group branch%{linkEnd} modification %{exceptSelection} matching the wildcard pattern: %{projectSelection}" +msgid "ScanResultPolicy|Prevent %{linkStart}group branch%{linkEnd} modification %{exceptSelection} matching the full paths: %{groupSelection}" msgstr "" msgid "ScanResultPolicy|Prevent approval by commit author" @@ -49628,7 +49628,7 @@ msgstr "" msgid "SecurityOrchestration|Every time a pipeline runs for %{branches}%{branchExceptionsString}" msgstr "" -msgid "SecurityOrchestration|Ex: development/*" +msgid "SecurityOrchestration|Ex: top_level_group" msgstr "" msgid "SecurityOrchestration|Exception branches" @@ -50281,9 +50281,6 @@ msgstr "" msgid "SecurityOrchestration|default" msgstr "" -msgid "SecurityOrchestration|except branches" -msgstr "" - msgid "SecurityOrchestration|except groups" msgstr "" diff --git a/spec/controllers/projects/compare_controller_spec.rb b/spec/controllers/projects/compare_controller_spec.rb index 933258f5b66..15dbd417c99 100644 --- a/spec/controllers/projects/compare_controller_spec.rb +++ b/spec/controllers/projects/compare_controller_spec.rb @@ -726,4 +726,37 @@ RSpec.describe Projects::CompareController, feature_category: :source_code_manag end end end + + describe 'GET #rapid_diffs' do + subject(:send_request) { get :rapid_diffs, params: request_params } + + let(:format) { :html } + let(:request_params) do + { + namespace_id: project.namespace, + project_id: project, + from: '6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9', + to: '5937ac0a7beb003549fc5fd26fc247adbce4a52e' + } + end + + it 'renders rapid_diffs template' do + send_request + + expect(assigns(:diffs).diff_files.first).to be_present + expect(response).to render_template(:rapid_diffs) + end + + context 'when the feature flag rapid_diffs is disabled' do + before do + stub_feature_flags(rapid_diffs: false) + end + + it 'returns 404' do + send_request + + expect(response).to have_gitlab_http_status(:not_found) + end + end + end end diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb index 7a9e9afd88d..3627a88f460 100644 --- a/spec/factories/projects.rb +++ b/spec/factories/projects.rb @@ -655,4 +655,10 @@ FactoryBot.define do project.namespace.namespace_settings.update!(allow_runner_registration_token: true) end end + + trait :import_user_mapping_enabled do + import_data_attributes do + { data: { user_contribution_mapping_enabled: true } } + end + end end diff --git a/spec/lib/gitlab/background_migration/fix_pick_up_at_ci_deleted_object_spec.rb b/spec/lib/gitlab/background_migration/fix_pick_up_at_ci_deleted_object_spec.rb index ec700a25781..f2dcd89f673 100644 --- a/spec/lib/gitlab/background_migration/fix_pick_up_at_ci_deleted_object_spec.rb +++ b/spec/lib/gitlab/background_migration/fix_pick_up_at_ci_deleted_object_spec.rb @@ -41,7 +41,7 @@ RSpec.describe Gitlab::BackgroundMigration::FixPickUpAtCiDeletedObject, schema: describe '#perform' do context 'when there are invalid records' do - it 'resets pick_up_at values' do + it 'resets pick_up_at values', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/500119' do expect { migration.perform } .to not_change { deleted_object1.reload.pick_up_at } .and not_change { deleted_object2.reload.pick_up_at } diff --git a/spec/lib/gitlab/legacy_github_import/base_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/base_formatter_spec.rb index 88279b8a210..4ad366410ac 100644 --- a/spec/lib/gitlab/legacy_github_import/base_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/base_formatter_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::BaseFormatter, feature_category: :importers do let_it_be(:project) { create(:project, import_type: 'gitea', namespace: create(:namespace, path: 'octocat')) } - let(:client) { double } + let(:client) { instance_double(Gitlab::LegacyGithubImport::Client) } let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') } @@ -58,4 +58,10 @@ RSpec.describe Gitlab::LegacyGithubImport::BaseFormatter, feature_category: :imp end end end + + describe '#contributing_user_formatters' do + it 'must be implemented in subclasses' do + expect { base.contributing_user_formatters }.to raise_error(NotImplementedError) + end + end end diff --git a/spec/lib/gitlab/legacy_github_import/comment_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/comment_formatter_spec.rb index 780a9d1d19e..c472f1a8516 100644 --- a/spec/lib/gitlab/legacy_github_import/comment_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/comment_formatter_spec.rb @@ -2,10 +2,32 @@ require 'spec_helper' -RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter, feature_category: :importers do - let_it_be(:project) { create(:project, import_type: 'gitea') } - let(:client) { double } - let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } +RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter, :clean_gitlab_redis_shared_state, feature_category: :importers do + let_it_be(:project) do + create(:project, :with_import_url, :import_user_mapping_enabled, import_type: ::Import::SOURCE_GITEA) + end + + let_it_be(:source_user_mapper) do + Gitlab::Import::SourceUserMapper.new( + namespace: project.root_ancestor, + import_type: project.import_type, + source_hostname: 'https://gitea.com' + ) + end + + let_it_be(:octocat) { { id: 1234, login: 'octocat', full_name: 'Cat', email: 'octocat@example.com' } } + let_it_be(:import_source_user) do + create( + :import_source_user, + source_user_identifier: octocat[:id], + namespace: project.root_ancestor, + source_hostname: 'https://gitea.com', + import_type: ::Import::SOURCE_GITEA + ) + end + + let(:client) { instance_double(Gitlab::LegacyGithubImport::Client) } + let(:ghost_user) { { id: -1, login: 'Ghost' } } let(:created_at) { DateTime.strptime('2013-04-10T20:09:31Z') } let(:updated_at) { DateTime.strptime('2014-03-03T18:58:10Z') } let(:imported_from) { ::Import::SOURCE_GITEA } @@ -21,23 +43,48 @@ RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter, feature_category: : } end - subject(:comment) { described_class.new(project, raw, client) } + subject(:comment) { described_class.new(project, raw, client, source_user_mapper) } before do allow(client).to receive(:user).and_return(octocat) end describe '#attributes' do + context 'when the note author exists on the source' do + let(:raw) { base } + + it 'sets the note author to a placeholder user' do + expect(comment.attributes.fetch(:author_id)).to eq(import_source_user.placeholder_user_id) + end + + it 'returns note without created at tag line' do + expect(comment.attributes.fetch(:note)).to eq("I'm having a problem with this.") + end + end + + context 'when the note author has been deleted from Gitea' do + let(:ghost_user) { { id: -1, login: 'Ghost', email: 'ghost_user@gitea_import_dummy_email.com' } } + let(:raw) { base.merge(user: ghost_user) } + + it 'sets the note author as the project creator' do + expect(comment.attributes.fetch(:author_id)).to eq(project.creator_id) + end + + it 'returns note with "Created by:" tag line' do + expect(comment.attributes.fetch(:note)).to eq("*Created by: Ghost*\n\nI'm having a problem with this.") + end + end + context 'when do not reference a portion of the diff' do let(:raw) { base } it 'returns formatted attributes' do expected = { project: project, - note: "*Created by: octocat*\n\nI'm having a problem with this.", + note: "I'm having a problem with this.", commit_id: nil, line_code: nil, - author_id: project.creator_id, + author_id: import_source_user.placeholder_user_id, type: nil, created_at: created_at, updated_at: updated_at, @@ -63,10 +110,10 @@ RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter, feature_category: : it 'returns formatted attributes' do expected = { project: project, - note: "*Created by: octocat*\n\nGreat stuff", + note: "Great stuff", commit_id: '6dcb09b5b57875f334f61aebed695e2e4193db5e', line_code: 'ce1be0ff4065a6e9415095c95f25f47a633cef2b_4_3', - author_id: project.creator_id, + author_id: import_source_user.placeholder_user_id, type: 'LegacyDiffNote', created_at: created_at, updated_at: updated_at, @@ -77,37 +124,38 @@ RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter, feature_category: : end end - context 'when author is a GitLab user' do - let(:raw) { base.merge(user: octocat) } - - it 'returns GitLab user id associated with GitHub email as author_id' do - gl_user = create(:user, email: octocat[:email]) - - expect(comment.attributes.fetch(:author_id)).to eq gl_user.id - end - - it 'returns note without created at tag line' do - create(:user, email: octocat[:email]) - - expect(comment.attributes.fetch(:note)).to eq("I'm having a problem with this.") - end - end - context 'when importing a GitHub project' do + let_it_be(:project) do + create(:project, :with_import_url, :import_user_mapping_enabled, import_type: ::Import::SOURCE_GITHUB) + end + + let_it_be(:source_user_mapper) do + Gitlab::Import::SourceUserMapper.new( + namespace: project.root_ancestor, + import_type: project.import_type, + source_hostname: 'https://github.com' + ) + end + let(:imported_from) { ::Import::SOURCE_GITHUB } let(:raw) { base } - - before do - project.import_type = 'github' + let!(:import_source_user) do + create( + :import_source_user, + source_user_identifier: octocat[:id], + namespace: project.root_ancestor, + source_hostname: 'https://github.com', + import_type: imported_from + ) end it 'returns formatted attributes' do expected = { project: project, - note: "*Created by: octocat*\n\nI'm having a problem with this.", + note: "I'm having a problem with this.", commit_id: nil, line_code: nil, - author_id: project.creator_id, + author_id: import_source_user.placeholder_user_id, type: nil, created_at: created_at, updated_at: updated_at, @@ -117,5 +165,117 @@ RSpec.describe Gitlab::LegacyGithubImport::CommentFormatter, feature_category: : expect(comment.attributes).to eq(expected) end end + + context 'when a gitlab issuable record is assigned' do + let(:raw) { base } + let(:issuable) { create(:issue, project: project) } + + it 'saves the comment to the issuable' do + comment.gitlab_issuable = issuable + + expect { comment.create! }.to change { issuable.notes.count }.from(0).to(1) + end + end + + context 'when user contribution mapping is disabled' do + let(:raw) { base.merge(user: octocat) } + + before do + allow(project).to receive_message_chain(:import_data, :user_mapping_enabled?).and_return(false) + end + + context 'when author is a GitLab user' do + let_it_be(:gitlab_user) { create(:user, email: octocat[:email]) } + + it 'returns GitLab user id associated with GitHub email as author_id' do + expect(comment.attributes.fetch(:author_id)).to eq(gitlab_user.id) + end + + it 'returns note without created at tag line' do + expect(comment.attributes.fetch(:note)).to eq("I'm having a problem with this.") + end + end + + context 'when the author does not exist in gitlab' do + it 'sets the note author as the project creator' do + expect(comment.attributes.fetch(:author_id)).to eq(project.creator_id) + end + + it 'returns note with "Created by:" tag line' do + expect(comment.attributes.fetch(:note)).to eq("*Created by: octocat*\n\nI'm having a problem with this.") + end + + it 'does not create a placeholder user' do + expect { comment }.not_to change { User.where(user_type: :placeholder).count } + end + end + end + end + + describe '#project_association' do + let(:raw) { base } + + it { expect(comment.project_association).to eq(:notes) } + end + + describe '#contributing_user_formatters' do + let(:raw) { base } + + it 'returns a hash containing UserFormatters for user references in attributes' do + expect(comment.contributing_user_formatters).to match( + a_hash_including({ author_id: a_kind_of(Gitlab::LegacyGithubImport::UserFormatter) }) + ) + end + + it 'includes all user reference columns in #attributes' do + expect(comment.contributing_user_formatters.keys).to match_array( + comment.attributes.keys & Gitlab::ImportExport::Base::RelationFactory::USER_REFERENCES.map(&:to_sym) + ) + end + end + + describe '#create!', :aggregate_failures do + let(:issuable) { create(:issue, project: project) } + let(:raw) { base } + let(:store) do + Import::PlaceholderReferences::Store.new(import_source: imported_from, import_uid: project.import_state.id) + end + + before do + comment.gitlab_issuable = issuable + end + + it 'saves the comment' do + expect { comment.create! }.to change { issuable.notes.count }.from(0).to(1) + end + + it 'pushes placeholder references for comments made by existing users in Gitea' do + comment.create! + cached_references = store.get(100).map { |ref| Import::SourceUserPlaceholderReference.from_serialized(ref) } + + expect(cached_references.map(&:model)).to eq(['Note']) + expect(cached_references.map(&:source_user_id)).to eq([import_source_user.id]) + expect(cached_references.map(&:user_reference_column)).to match_array(['author_id']) + end + + context 'when the comment was made by a deleted user in Gitea' do + let(:raw) { base.merge(user: ghost_user) } + + it 'does not push any placeholder references' do + comment.create! + expect(store).to be_empty + end + end + + context 'when user contribution mapping is disabled' do + before do + allow(project).to receive_message_chain(:import_data, :user_mapping_enabled?).and_return(false) + end + + it 'does not push any placeholder references' do + comment.create! + expect(store).to be_empty + end + end end end diff --git a/spec/lib/gitlab/legacy_github_import/importer_spec.rb b/spec/lib/gitlab/legacy_github_import/importer_spec.rb index 7588500e7d2..ea754b807d8 100644 --- a/spec/lib/gitlab/legacy_github_import/importer_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/importer_spec.rb @@ -2,243 +2,24 @@ require 'spec_helper' -RSpec.describe Gitlab::LegacyGithubImport::Importer, feature_category: :importers do +RSpec.describe Gitlab::LegacyGithubImport::Importer, :clean_gitlab_redis_shared_state, feature_category: :importers do subject(:importer) { described_class.new(project) } - shared_examples 'Gitlab::LegacyGithubImport::Importer#execute' do - let(:expected_not_called) { [] } - - before do - allow(project).to receive(:import_data).and_return(double.as_null_object) - end - - it 'calls import methods' do - expected_called = [ - :import_labels, :import_milestones, :import_pull_requests, :import_issues, - :import_wiki, :import_releases, :handle_errors, - [:import_comments, :issues], - [:import_comments, :pull_requests] - ] - - expected_called -= expected_not_called - - aggregate_failures do - expected_called.each do |method_name, arg| - base_expectation = proc { expect(importer).to receive(method_name) } - arg ? base_expectation.call.with(arg) : base_expectation.call - end - - expected_not_called.each do |method_name, arg| - base_expectation = proc { expect(importer).not_to receive(method_name) } - arg ? base_expectation.call.with(arg) : base_expectation.call - end - end - - importer.execute - end + let_it_be(:api_root) { 'https://try.gitea.io/api/v1' } + let_it_be(:repo_root) { 'https://try.gitea.io' } + let_it_be(:project) do + create( + :project, :repository, :wiki_disabled, :import_user_mapping_enabled, + import_url: "#{repo_root}/foo/group/project.git", + import_type: ::Import::SOURCE_GITEA + ) end - shared_examples 'Gitlab::LegacyGithubImport::Importer#execute an error occurs' do - before do - allow(project).to receive(:import_data).and_return(double.as_null_object) - - allow(Rails).to receive(:cache).and_return(ActiveSupport::Cache::MemoryStore.new) - - allow_any_instance_of(Octokit::Client).to receive(:rate_limit!).and_raise(Octokit::NotFound) - allow(project.wiki.repository).to receive(:import_repository).and_raise(Gitlab::Git::CommandError) - - allow_any_instance_of(Octokit::Client).to receive(:user).and_return(octocat) - allow_any_instance_of(Octokit::Client).to receive(:labels).and_return([label1, label2]) - allow_any_instance_of(Octokit::Client).to receive(:milestones).and_return([milestone, milestone]) - allow_any_instance_of(Octokit::Client).to receive(:issues).and_return([issue1, issue2]) - allow_any_instance_of(Octokit::Client).to receive(:pull_requests).and_return([pull_request, pull_request_missing_source_branch]) - allow_any_instance_of(Octokit::Client).to receive(:issues_comments).and_raise(Octokit::NotFound) - allow_any_instance_of(Octokit::Client).to receive(:pull_requests_comments).and_return([]) - allow_any_instance_of(Octokit::Client).to receive(:last_response).and_return(double(rels: { next: nil })) - allow_any_instance_of(Octokit::Client).to receive(:releases).and_return([release1, release2]) - - allow(importer).to receive(:restore_source_branch).and_raise(StandardError, 'Some error') - end - - let(:label1) do - { - name: 'Bug', - color: 'ff0000', - url: "#{api_root}/repos/octocat/Hello-World/labels/bug" - } - end - - let(:label2) do - { - name: nil, - color: 'ff0000', - url: "#{api_root}/repos/octocat/Hello-World/labels/bug" - } - end - - let(:milestone) do - { - id: 1347, # For Gitea - number: 1347, - state: 'open', - title: '1.0', - description: 'Version 1.0', - due_on: nil, - created_at: created_at, - updated_at: updated_at, - closed_at: nil, - url: "#{api_root}/repos/octocat/Hello-World/milestones/1" - } - end - - let(:issue1) do - { - number: 1347, - milestone: nil, - state: 'open', - title: 'Found a bug', - body: "I'm having a problem with this.", - assignee: nil, - user: octocat, - comments: 0, - pull_request: nil, - created_at: created_at, - updated_at: updated_at, - closed_at: nil, - url: "#{api_root}/repos/octocat/Hello-World/issues/1347", - labels: [{ name: 'Label #1' }] - } - end - - let(:issue2) do - { - number: 1348, - milestone: nil, - state: 'open', - title: nil, - body: "I'm having a problem with this.", - assignee: nil, - user: octocat, - comments: 0, - pull_request: nil, - created_at: created_at, - updated_at: updated_at, - closed_at: nil, - url: "#{api_root}/repos/octocat/Hello-World/issues/1348", - labels: [{ name: 'Label #2' }] - } - end - - let(:release1) do - { - tag_name: 'v1.0.0', - name: 'First release', - body: 'Release v1.0.0', - draft: false, - created_at: created_at, - published_at: created_at, - updated_at: updated_at, - url: "#{api_root}/repos/octocat/Hello-World/releases/1" - } - end - - let(:release2) do - { - tag_name: 'v1.1.0', - name: 'Second release', - body: nil, - draft: false, - created_at: created_at, - published_at: created_at, - updated_at: updated_at, - url: "#{api_root}/repos/octocat/Hello-World/releases/2" - } - end - - it 'returns true' do - expect(subject.execute).to eq true - end - - it 'does not raise an error' do - expect { subject.execute }.not_to raise_error - end - - it 'stores error messages', :unlimited_max_formatted_output_length do - error = { - message: 'The remote data could not be fully imported.', - errors: [ - { type: :label, url: "#{api_root}/repos/octocat/Hello-World/labels/bug", errors: "Validation failed: Title can't be blank, Title is invalid" }, - { type: :pull_request, url: "#{api_root}/repos/octocat/Hello-World/pulls/1347", errors: 'Some error' }, - { type: :issue, url: "#{api_root}/repos/octocat/Hello-World/issues/1348", errors: "Validation failed: Title can't be blank" }, - { type: :issues_comments, errors: 'Octokit::NotFound' }, - { type: :wiki, errors: "Gitlab::Git::CommandError" } - ] - } - - importer.execute - - expect(project.import_state.last_error).to eq error.to_json - end - - context 'when comment has invalid created date' do - let(:comment_with_invalid_date) do - { - html_url: "#{api_root}/repos/octocat/Hello-World/issues/1347", - body: "I'm having a problem with this.", - user: octocat, - commit_id: nil, - diff_hunk: nil, - created_at: DateTime.strptime('1900-01-26T19:01:12Z'), - updated_at: updated_at - } - end - - before do - allow_any_instance_of(Octokit::Client).to receive(:issues_comments).and_return([comment_with_invalid_date]) - end - - it 'stores error messages' do - importer.execute - - expect(Gitlab::Json.parse(project.import_state.last_error)).to include({ - 'errors' => include( - { "errors" => "Validation failed: Created at The created date provided is too far in the past.", "type" => "comment", "url" => "#{api_root}/repos/octocat/Hello-World/issues/1347" } - ) - }) - end - end - end - - shared_examples 'Gitlab::LegacyGithubImport unit-testing' do - describe '#clean_up_restored_branches' do - before do - allow(gh_pull_request).to receive(:source_branch_exists?).at_least(:once) { false } - allow(gh_pull_request).to receive(:target_branch_exists?).at_least(:once) { false } - end - - context 'when pull request stills open' do - let(:gh_pull_request) { Gitlab::LegacyGithubImport::PullRequestFormatter.new(project, pull_request) } - - it 'does not remove branches' do - expect(subject).not_to receive(:remove_branch) - subject.send(:clean_up_restored_branches, gh_pull_request) - end - end - - context 'when pull request is closed' do - let(:gh_pull_request) { Gitlab::LegacyGithubImport::PullRequestFormatter.new(project, closed_pull_request) } - - it 'does remove branches' do - expect(subject).to receive(:remove_branch).at_least(:twice) - subject.send(:clean_up_restored_branches, gh_pull_request) - end - end - end - end - - let(:project) { create(:project, :repository, :wiki_disabled, import_url: "#{repo_root}/octocat/Hello-World.git") } let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } let(:credentials) { { user: 'joe' } } + let(:store) do + Import::PlaceholderReferences::Store.new(import_source: project.import_type, import_uid: project.import_state.id) + end let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') } @@ -297,20 +78,304 @@ RSpec.describe Gitlab::LegacyGithubImport::Importer, feature_category: :importer } end - context 'when importing a Gitea project' do - let(:api_root) { 'https://try.gitea.io/api/v1' } - let(:repo_root) { 'https://try.gitea.io' } + let(:label1) do + { + name: 'Bug', + color: 'ff0000', + url: "#{api_root}/repos/octocat/Hello-World/labels/bug" + } + end + let(:label2) do + { + name: nil, + color: 'ff0000', + url: "#{api_root}/repos/octocat/Hello-World/labels/bug" + } + end + + let(:milestone) do + { + id: 1347, # For Gitea + number: 1347, + state: 'open', + title: '1.0', + description: 'Version 1.0', + due_on: nil, + created_at: created_at, + updated_at: updated_at, + closed_at: nil, + url: "#{api_root}/repos/octocat/Hello-World/milestones/1" + } + end + + let(:issue1) do + { + number: 1347, + milestone: nil, + state: 'open', + title: 'Found a bug', + body: "I'm having a problem with this.", + assignee: nil, + user: octocat, + comments: 0, + pull_request: nil, + created_at: created_at, + updated_at: updated_at, + closed_at: nil, + url: "#{api_root}/repos/octocat/Hello-World/issues/1347", + labels: [{ name: 'Label #1' }] + } + end + + let(:issue2) do + { + number: 1348, + milestone: nil, + state: 'open', + title: nil, + body: "I'm having a problem with this.", + assignee: nil, + user: octocat, + comments: 0, + pull_request: nil, + created_at: created_at, + updated_at: updated_at, + closed_at: nil, + url: "#{api_root}/repos/octocat/Hello-World/issues/1348", + labels: [{ name: 'Label #2' }] + } + end + + let(:release1) do + { + tag_name: 'v1.0.0', + name: 'First release', + body: 'Release v1.0.0', + draft: false, + created_at: created_at, + published_at: created_at, + updated_at: updated_at, + url: "#{api_root}/repos/octocat/Hello-World/releases/1" + } + end + + let(:release2) do + { + tag_name: 'v1.1.0', + name: 'Second release', + body: nil, + draft: false, + created_at: created_at, + published_at: created_at, + updated_at: updated_at, + url: "#{api_root}/repos/octocat/Hello-World/releases/2" + } + end + + describe '#execute' do before do - project.update!(import_type: 'gitea', import_url: "#{repo_root}/foo/group/project.git") + allow(Import::PlaceholderReferences::Store).to receive(:new).and_return(store) + allow(store).to receive(:empty?).and_return(true) + + # Lower wait and timeout limit to make spec faster + stub_const("#{described_class}::PLACEHOLDER_LOAD_SLEEP", 0.01) + stub_const("#{described_class}::PLACEHOLDER_LOAD_TIMEOUT", 0.05) end - it_behaves_like 'Gitlab::LegacyGithubImport::Importer#execute' do - let(:expected_not_called) { [:import_releases, [:import_comments, :pull_requests]] } + context 'with stages' do + before do + allow(project).to receive(:import_data).and_return(double.as_null_object) + allow(project).to receive_message_chain(:wiki, :repository_exists?).and_return(true) + allow(importer).to receive(:fetch_resources).and_return(nil) + end + + it 'calls gitea importer stages', :aggregate_failures do + expect(importer).to receive(:import_labels).and_call_original + expect(importer).to receive(:import_milestones).and_call_original + expect(importer).to receive(:import_pull_requests).and_call_original + expect(importer).to receive(:import_issues).and_call_original + expect(importer).to receive(:import_wiki).and_call_original + expect(importer).to receive(:handle_errors).and_call_original + expect(importer).to receive(:import_comments).with(:issues).and_call_original + + importer.execute + end + + it 'does not call github-specific importer stages', :aggregate_failures do + expect(importer).not_to receive(:import_releases) + expect(importer).not_to receive(:import_comments).with(:pull_requests) + + importer.execute + end + + it 'loads placeholder references after each relevant stage' do + # import_wiki does not load placeholder references because it doesn't have any user attributes to map + # handle_errors does not create GitLab records + stages_that_push_placeholder_references = [ + :import_pull_requests, :import_issues, :import_comments + ] + + expect(::Import::LoadPlaceholderReferencesWorker).to receive(:perform_async).exactly( + stages_that_push_placeholder_references.length + ).times.with( + project.import_type, + project.import_state.id, + 'current_user_id' => project.creator_id + ) + + importer.execute + end + + it 'waits for the placeholder references to be loaded from the store without error' do + allow(store).to receive(:empty?).and_return(false, false, false, false, true) + + expect(Kernel).to receive(:sleep).with(0.01).exactly(4).times + + importer.execute + + expect(Gitlab::Json.parse(project.import_state.last_error)).to be_nil + end + + it 'times out and logs an error when references fail to load' do + allow(store).to receive(:empty?).and_return(false) + + expect(Kernel).to receive(:sleep).with(0.01).exactly(5).times + + importer.execute + + expect(Gitlab::Json.parse(project.import_state.last_error)).to include({ + 'errors' => include( + { + 'type' => 'placeholder_references', + 'errors' => "Timed out after waiting #{described_class::PLACEHOLDER_LOAD_TIMEOUT} seconds " \ + "for placeholder references to finish saving" + } + ) + }) + end + + context 'when user contribution mapping is disabled' do + before do + allow(project).to receive_message_chain(:import_data, :user_mapping_enabled?).and_return(false) + end + + it 'does not enqueue the worker to load placeholder references' do + expect(Import::LoadPlaceholderReferencesWorker).not_to receive(:perform_async) + + importer.execute + end + + it 'does not sleep' do + allow(store).to receive(:empty?).and_return(false) + + expect(Kernel).not_to receive(:sleep) + + importer.execute + end + end end - it_behaves_like 'Gitlab::LegacyGithubImport::Importer#execute an error occurs' - it_behaves_like 'Gitlab::LegacyGithubImport unit-testing' + context 'when an error occurs' do + before do + allow(project).to receive(:import_data).and_return(double.as_null_object) + + allow(Rails).to receive(:cache).and_return(ActiveSupport::Cache::MemoryStore.new) + + allow_any_instance_of(Octokit::Client).to receive(:rate_limit!).and_raise(Octokit::NotFound) + allow(project.wiki.repository).to receive(:import_repository).and_raise(Gitlab::Git::CommandError) + + allow_any_instance_of(Octokit::Client).to receive(:user).and_return(octocat) + allow_any_instance_of(Octokit::Client).to receive(:labels).and_return([label1, label2]) + allow_any_instance_of(Octokit::Client).to receive(:milestones).and_return([milestone, milestone]) + allow_any_instance_of(Octokit::Client).to receive(:issues).and_return([issue1, issue2]) + allow_any_instance_of(Octokit::Client).to receive(:pull_requests).and_return([pull_request, pull_request_missing_source_branch]) + allow_any_instance_of(Octokit::Client).to receive(:issues_comments).and_raise(Octokit::NotFound) + allow_any_instance_of(Octokit::Client).to receive(:pull_requests_comments).and_return([]) + allow_any_instance_of(Octokit::Client).to receive(:last_response).and_return(double(rels: { next: nil })) + allow_any_instance_of(Octokit::Client).to receive(:releases).and_return([release1, release2]) + + allow(importer).to receive(:restore_source_branch).and_raise(StandardError, 'Some error') + end + + it 'returns true' do + expect(subject.execute).to eq true + end + + it 'does not raise an error' do + expect { subject.execute }.not_to raise_error + end + + it 'stores error messages', :unlimited_max_formatted_output_length do + error = { + message: 'The remote data could not be fully imported.', + errors: [ + { type: :label, url: "#{api_root}/repos/octocat/Hello-World/labels/bug", errors: "Validation failed: Title can't be blank, Title is invalid" }, + { type: :pull_request, url: "#{api_root}/repos/octocat/Hello-World/pulls/1347", errors: 'Some error' }, + { type: :issue, url: "#{api_root}/repos/octocat/Hello-World/issues/1348", errors: "Validation failed: Title can't be blank" }, + { type: :issues_comments, errors: 'Octokit::NotFound' }, + { type: :wiki, errors: "Gitlab::Git::CommandError" } + ] + } + + importer.execute + + expect(project.import_state.last_error).to eq error.to_json + end + + context 'when comment has invalid created date' do + let(:comment_with_invalid_date) do + { + html_url: "#{api_root}/repos/octocat/Hello-World/issues/1347", + body: "I'm having a problem with this.", + user: octocat, + commit_id: nil, + diff_hunk: nil, + created_at: DateTime.strptime('1900-01-26T19:01:12Z'), + updated_at: updated_at + } + end + + before do + allow_any_instance_of(Octokit::Client).to receive(:issues_comments).and_return([comment_with_invalid_date]) + end + + it 'stores error messages' do + importer.execute + + expect(Gitlab::Json.parse(project.import_state.last_error)).to include({ + 'errors' => include( + { "errors" => "Validation failed: Created at The created date provided is too far in the past.", "type" => "comment", "url" => "#{api_root}/repos/octocat/Hello-World/issues/1347" } + ) + }) + end + end + end + + describe '#clean_up_restored_branches' do + before do + allow(gh_pull_request).to receive(:source_branch_exists?).at_least(:once) { false } + allow(gh_pull_request).to receive(:target_branch_exists?).at_least(:once) { false } + end + + context 'when pull request stills open' do + let(:gh_pull_request) { Gitlab::LegacyGithubImport::PullRequestFormatter.new(project, pull_request) } + + it 'does not remove branches' do + expect(subject).not_to receive(:remove_branch) + subject.send(:clean_up_restored_branches, gh_pull_request) + end + end + + context 'when pull request is closed' do + let(:gh_pull_request) { Gitlab::LegacyGithubImport::PullRequestFormatter.new(project, closed_pull_request) } + + it 'does remove branches' do + expect(subject).to receive(:remove_branch).at_least(:twice) + subject.send(:clean_up_restored_branches, gh_pull_request) + end + end + end describe '#client' do it 'instantiates a Client' do diff --git a/spec/lib/gitlab/legacy_github_import/issuable_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/issuable_formatter_spec.rb index 56a51c6bddd..6a8b6cf38ef 100644 --- a/spec/lib/gitlab/legacy_github_import/issuable_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/issuable_formatter_spec.rb @@ -14,6 +14,10 @@ RSpec.describe Gitlab::LegacyGithubImport::IssuableFormatter do it { expect { issuable_formatter.project_association }.to raise_error(NotImplementedError) } end + describe '#project_assignee_association' do + it { expect { issuable_formatter.project_assignee_association }.to raise_error(NotImplementedError) } + end + describe '#number' do it { expect(issuable_formatter.number).to eq(42) } end @@ -21,4 +25,8 @@ RSpec.describe Gitlab::LegacyGithubImport::IssuableFormatter do describe '#find_condition' do it { expect(issuable_formatter.find_condition).to eq({ iid: 42 }) } end + + describe '#contributing_assignee_formatters' do + it { expect { issuable_formatter.contributing_assignee_formatters }.to raise_error(NotImplementedError) } + end end diff --git a/spec/lib/gitlab/legacy_github_import/issue_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/issue_formatter_spec.rb index 4427f0163c8..6efc3fcb4bb 100644 --- a/spec/lib/gitlab/legacy_github_import/issue_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/issue_formatter_spec.rb @@ -2,14 +2,41 @@ require 'spec_helper' -RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter, feature_category: :importers do - let_it_be(:project) { create(:project, import_type: 'gitea', namespace: create(:namespace, path: 'octocat')) } - let(:client) { double } - let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } +RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter, :clean_gitlab_redis_shared_state, feature_category: :importers do + let_it_be(:project) do + create( + :project, + :with_import_url, + :import_user_mapping_enabled, + import_type: ::Import::SOURCE_GITEA, + namespace: create(:namespace, path: 'octocat') + ) + end + + let_it_be(:source_user_mapper) do + Gitlab::Import::SourceUserMapper.new( + namespace: project.root_ancestor, + import_type: project.import_type, + source_hostname: 'https://gitea.com' + ) + end + + let_it_be(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } + let_it_be(:import_source_user) do + create( + :import_source_user, + source_user_identifier: octocat[:id], + namespace: project.root_ancestor, + source_hostname: 'https://gitea.com', + import_type: ::Import::SOURCE_GITEA + ) + end + + let(:client) { instance_double(Gitlab::LegacyGithubImport::Client) } + let(:ghost_user) { { id: -1, login: 'Ghost' } } let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') } let(:imported_from) { ::Import::SOURCE_GITEA } - let(:base_data) do { number: 1347, @@ -27,7 +54,7 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter, feature_category: :im } end - subject(:issue) { described_class.new(project, raw_data, client) } + subject(:issue) { described_class.new(project, raw_data, client, source_user_mapper) } before do allow(client).to receive(:user).and_return(octocat) @@ -43,9 +70,9 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter, feature_category: :im project: project, milestone: nil, title: 'Found a bug', - description: "*Created by: octocat*\n\nI'm having a problem with this.", + description: "I'm having a problem with this.", state: 'opened', - author_id: project.creator_id, + author_id: import_source_user.placeholder_user_id, assignee_ids: [], created_at: created_at, updated_at: updated_at, @@ -65,9 +92,9 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter, feature_category: :im project: project, milestone: nil, title: 'Found a bug', - description: "*Created by: octocat*\n\nI'm having a problem with this.", + description: "I'm having a problem with this.", state: 'closed', - author_id: project.creator_id, + author_id: import_source_user.placeholder_user_id, assignee_ids: [], created_at: created_at, updated_at: updated_at, @@ -78,17 +105,57 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter, feature_category: :im end end - context 'when it is assigned to someone' do - let(:raw_data) { base_data.merge(assignee: octocat) } + context 'when it is assigned to a user' do + context 'and the assigned user has a placeholder user in gitlab' do + let(:raw_data) { base_data.merge(assignee: octocat) } - it 'returns nil as assignee_id when is not a GitLab user' do - expect(issue.attributes.fetch(:assignee_ids)).to be_empty + it 'returns an existing placeholder user id' do + expect(issue.attributes.fetch(:assignee_ids)).to eq([import_source_user.placeholder_user_id]) + end end - it 'returns GitLab user id associated with GitHub email as assignee_id' do - gl_user = create(:user, email: octocat[:email]) + context 'and the assigned user does not already have a placeholder user' do + let(:octocat_2) { { id: 999999, login: 'octocat two', email: 'octocat2@example.com' } } + let(:raw_data) { base_data.merge(assignee: octocat_2) } - expect(issue.attributes.fetch(:assignee_ids)).to eq [gl_user.id] + it 'creates and returns a new placeholder user id', :aggregate_failures do + assignee_id = issue.attributes.fetch(:assignee_ids).first + + expect(User.find(assignee_id).user_type).to eq('placeholder') + expect(assignee_id).not_to eq(import_source_user.placeholder_user_id) + end + end + + context 'and it is assigned to a deleted gitea user' do + let(:raw_data) { base_data.merge(assignee: ghost_user) } + + it 'returns nil for assignee_ids' do + expect(issue.attributes.fetch(:assignee_ids)).to be_empty + end + end + + context 'and user contribution mapping is disabled' do + let(:raw_data) { base_data.merge(assignee: octocat) } + + before do + allow(project).to receive_message_chain(:import_data, :user_mapping_enabled?).and_return(false) + end + + it 'returns nil as assignee_id when is not a GitLab user' do + expect(issue.attributes.fetch(:assignee_ids)).to be_empty + end + + it 'does not create any placeholder users' do + expect { issue.attributes.fetch(:assignee_ids) }.not_to change { + User.where(user_type: :placeholder).count + } + end + + it 'returns GitLab user id associated with Gitea email as assignee_id' do + gl_user = create(:user, email: octocat[:email]) + + expect(issue.attributes.fetch(:assignee_ids)).to eq [gl_user.id] + end end end @@ -107,23 +174,56 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter, feature_category: :im end end - context 'when author is a GitLab user' do - let(:raw_data) { base_data.merge(user: octocat) } + context 'when the issue has an author' do + context 'and the author has a placeholder user in gitlab' do + let(:raw_data) { base_data.merge(user: octocat) } - it 'returns project creator_id as author_id when is not a GitLab user' do - expect(issue.attributes.fetch(:author_id)).to eq project.creator_id + it 'returns an existing placeholder user id' do + expect(issue.attributes.fetch(:author_id)).to eq(import_source_user.placeholder_user_id) + end end - it 'returns GitLab user id associated with GitHub email as author_id' do - gl_user = create(:user, email: octocat[:email]) + context 'and the author does not already have a placeholder user' do + let(:octocat_2) { { id: 999999, login: 'octocat two', email: 'octocat2@example.com' } } + let(:raw_data) { base_data.merge(user: octocat_2) } - expect(issue.attributes.fetch(:author_id)).to eq gl_user.id + it 'creates and returns a new placeholder user id', :aggregate_failures do + author_id = issue.attributes.fetch(:author_id) + expect(User.find(author_id).user_type).to eq('placeholder') + expect(author_id).not_to eq(import_source_user.placeholder_user_id) + end end - it 'returns description without created at tag line' do - create(:user, email: octocat[:email]) + context 'and the author is a deleted gitea user' do + let(:raw_data) { base_data.merge(user: ghost_user) } - expect(issue.attributes.fetch(:description)).to eq("I'm having a problem with this.") + it 'returns the project creator id' do + expect(issue.attributes.fetch(:author_id)).to eq(project.creator_id) + end + end + + context 'and user contribution mapping is disabled' do + let(:raw_data) { base_data.merge(user: octocat) } + + before do + allow(project).to receive_message_chain(:import_data, :user_mapping_enabled?).and_return(false) + end + + it 'returns project creator_id as author_id when is not a GitLab user' do + expect(issue.attributes.fetch(:author_id)).to eq project.creator_id + end + + it 'returns GitLab user id associated with Gitea email as author_id' do + gl_user = create(:user, email: octocat[:email]) + + expect(issue.attributes.fetch(:author_id)).to eq gl_user.id + end + + it 'returns description without created at tag line' do + create(:user, email: octocat[:email]) + + expect(issue.attributes.fetch(:description)).to eq("I'm having a problem with this.") + end end end end @@ -142,12 +242,36 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter, feature_category: :im end context 'when importing a GitHub project' do - let(:imported_from) { ::Import::SOURCE_GITHUB } - - before do - project.import_type = 'github' + let_it_be(:project) do + create( + :project, + :with_import_url, + :import_user_mapping_enabled, + import_type: ::Import::SOURCE_GITHUB, + namespace: create(:namespace, path: 'octocat-github') + ) end + let_it_be(:source_user_mapper) do + Gitlab::Import::SourceUserMapper.new( + namespace: project.root_ancestor, + import_type: project.import_type, + source_hostname: 'https://github.com' + ) + end + + let_it_be(:import_source_user) do + create( + :import_source_user, + source_user_identifier: octocat[:id], + namespace: project.root_ancestor, + source_hostname: 'https://github.com', + import_type: ::Import::SOURCE_GITHUB + ) + end + + let(:imported_from) { ::Import::SOURCE_GITHUB } + it_behaves_like 'Gitlab::LegacyGithubImport::IssueFormatter#attributes' it_behaves_like 'Gitlab::LegacyGithubImport::IssueFormatter#number' end @@ -187,4 +311,101 @@ RSpec.describe Gitlab::LegacyGithubImport::IssueFormatter, feature_category: :im end end end + + describe '#project_association' do + let(:raw_data) { base_data } + + it { expect(issue.project_association).to eq(:issues) } + end + + describe '#project_assignee_association' do + let(:raw_data) { base_data } + + it { expect(issue.project_assignee_association).to eq(:issue_assignees) } + end + + describe '#contributing_user_formatters' do + let(:raw_data) { base_data } + + it 'returns a hash containing UserFormatters for user references in attributes' do + expect(issue.contributing_user_formatters).to match( + a_hash_including({ author_id: a_kind_of(Gitlab::LegacyGithubImport::UserFormatter) }) + ) + end + + it 'includes all user reference columns in #attributes' do + expect(issue.contributing_user_formatters.keys).to match_array( + issue.attributes.keys & Gitlab::ImportExport::Base::RelationFactory::USER_REFERENCES.map(&:to_sym) + ) + end + end + + describe '#contributing_assignee_formatters' do + let(:raw_data) { base_data.merge(assignee: octocat) } + + it 'returns a hash containing the author UserFormatter' do + expect(issue.contributing_assignee_formatters).to match( + a_hash_including({ user_id: a_kind_of(Gitlab::LegacyGithubImport::UserFormatter) }) + ) + end + end + + describe '#create!', :aggregate_failures do + let(:raw_data) { base_data.merge(assignee: octocat) } + let(:store) do + Import::PlaceholderReferences::Store.new(import_source: imported_from, import_uid: project.import_state.id) + end + + it 'saves the issue and assignees' do + issue.create! + created_issue = project.issues.find_by_iid(issue.attributes[:iid]) + + expect(created_issue).not_to be_nil + expect(created_issue&.issue_assignees).not_to be_empty + end + + it 'pushes placeholder references for user references on the issue' do + issue.create! + cached_references = store.get(100).filter_map do |item| + reference = Import::SourceUserPlaceholderReference.from_serialized(item) + reference if reference.model == 'Issue' + end + + expect(cached_references.map(&:model)).to eq(['Issue']) + expect(cached_references.map(&:source_user_id)).to eq([import_source_user.id]) + expect(cached_references.map(&:user_reference_column)).to eq(['author_id']) + end + + it 'pushes placeholder references for user references on the issue assignees' do + issue.create! + cached_references = store.get(100).filter_map do |item| + reference = Import::SourceUserPlaceholderReference.from_serialized(item) + reference if reference.model == 'IssueAssignee' + end + + expect(cached_references.map(&:model)).to match_array(['IssueAssignee']) + expect(cached_references.map(&:source_user_id)).to eq([import_source_user.id]) + expect(cached_references.map(&:user_reference_column)).to match_array(['user_id']) + end + + context 'when the issue references deleted users in Gitea' do + let(:raw_data) { base_data.merge(user: ghost_user, assignee: ghost_user) } + + it 'does not push any placeholder references' do + issue.create! + expect(store.empty?).to eq(true) + end + end + + context 'when user contribution mapping is disabled' do + before do + allow(project).to receive_message_chain(:import_data, :user_mapping_enabled?).and_return(false) + end + + it 'does not push any placeholder references' do + issue.create! + expect(store.empty?).to eq(true) + end + end + end end diff --git a/spec/lib/gitlab/legacy_github_import/label_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/label_formatter_spec.rb index 8e2c8031a6f..22232607570 100644 --- a/spec/lib/gitlab/legacy_github_import/label_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/label_formatter_spec.rb @@ -3,14 +3,15 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::LabelFormatter do - let_it_be(:project) { create(:project) } + let_it_be(:project) { create(:project, :with_import_url, :import_user_mapping_enabled) } + let(:raw) { { name: 'improvements', color: 'e6e6e6' } } - subject { described_class.new(project, raw) } + subject(:label) { described_class.new(project, raw) } describe '#attributes' do it 'returns formatted attributes' do - expect(subject.attributes).to eq({ + expect(label.attributes).to eq({ project: project, title: 'improvements', color: '#e6e6e6' @@ -18,19 +19,40 @@ RSpec.describe Gitlab::LegacyGithubImport::LabelFormatter do end end - describe '#create!' do - context 'when label does not exist' do - it 'creates a new label' do - expect { subject.create! }.to change(Label, :count).by(1) - end + describe '#contributing_user_formatters' do + it { expect(label.contributing_user_formatters).to eq({}) } + + it 'includes all user reference columns in #attributes' do + expect(label.contributing_user_formatters.keys).to match_array( + label.attributes.keys & Gitlab::ImportExport::Base::RelationFactory::USER_REFERENCES.map(&:to_sym) + ) + end + end + + describe '#create!', :aggregate_failures, :clean_gitlab_redis_shared_state do + let(:store) do + Import::PlaceholderReferences::Store.new( + import_source: ::Import::SOURCE_GITHUB, + import_uid: project.import_state.id + ) end - context 'when label exists' do - it 'does not create a new label' do - Labels::CreateService.new(name: raw[:name]).execute(project: project) + it 'creates a new label when label does not exist' do + expect { label.create! }.to change(Label, :count).by(1) + end - expect { subject.create! }.not_to change(Label, :count) - end + it 'does not create a new label when label exists' do + Labels::CreateService.new(name: raw[:name]).execute(project: project) + + expect { label.create! }.not_to change(Label, :count) + end + + it 'does not push any placeholder references because it does not reference a user' do + label_user_references = label.attributes.keys & Gitlab::ImportExport::Base::RelationFactory::USER_REFERENCES + label.create! + + expect(store.empty?).to be(true) + expect(label_user_references).to be_empty end end end diff --git a/spec/lib/gitlab/legacy_github_import/milestone_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/milestone_formatter_spec.rb index 7c57bf9c707..631347fb85e 100644 --- a/spec/lib/gitlab/legacy_github_import/milestone_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/milestone_formatter_spec.rb @@ -3,7 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::MilestoneFormatter do - let_it_be(:project) { create(:project) } + let_it_be(:project) { create(:project, :with_import_url, :import_user_mapping_enabled) } + let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') } let(:base_data) do @@ -20,81 +21,117 @@ RSpec.describe Gitlab::LegacyGithubImport::MilestoneFormatter do let(:iid_attr) { :number } - subject(:formatter) { described_class.new(project, raw_data) } + subject(:milestone) { described_class.new(project, raw_data) } - shared_examples 'Gitlab::LegacyGithubImport::MilestoneFormatter#attributes' do - let(:data) { base_data.merge(iid_attr => 1347) } + describe '#attributes' do + shared_examples 'Gitlab::LegacyGithubImport::MilestoneFormatter#attributes' do + let(:data) { base_data.merge(iid_attr => 1347) } - context 'when milestone is open' do - let(:raw_data) { data.merge(state: 'open') } + context 'when milestone is open' do + let(:raw_data) { data.merge(state: 'open') } - it 'returns formatted attributes' do - expected = { - iid: 1347, - project: project, - title: '1.0', - description: 'Version 1.0', - state: 'active', - due_date: nil, - created_at: created_at, - updated_at: updated_at - } + it 'returns formatted attributes' do + expected = { + iid: 1347, + project: project, + title: '1.0', + description: 'Version 1.0', + state: 'active', + due_date: nil, + created_at: created_at, + updated_at: updated_at + } - expect(formatter.attributes).to eq(expected) + expect(milestone.attributes).to eq(expected) + end + end + + context 'when milestone is closed' do + let(:raw_data) { data.merge(state: 'closed') } + + it 'returns formatted attributes' do + expected = { + iid: 1347, + project: project, + title: '1.0', + description: 'Version 1.0', + state: 'closed', + due_date: nil, + created_at: created_at, + updated_at: updated_at + } + + expect(milestone.attributes).to eq(expected) + end + end + + context 'when milestone has a due date' do + let(:due_date) { DateTime.strptime('2011-01-28T19:01:12Z') } + let(:raw_data) { data.merge(due_on: due_date) } + + it 'returns formatted attributes' do + expected = { + iid: 1347, + project: project, + title: '1.0', + description: 'Version 1.0', + state: 'active', + due_date: due_date, + created_at: created_at, + updated_at: updated_at + } + + expect(milestone.attributes).to eq(expected) + end end end - context 'when milestone is closed' do - let(:raw_data) { data.merge(state: 'closed') } - - it 'returns formatted attributes' do - expected = { - iid: 1347, - project: project, - title: '1.0', - description: 'Version 1.0', - state: 'closed', - due_date: nil, - created_at: created_at, - updated_at: updated_at - } - - expect(formatter.attributes).to eq(expected) - end + context 'when importing a GitHub project' do + it_behaves_like 'Gitlab::LegacyGithubImport::MilestoneFormatter#attributes' end - context 'when milestone has a due date' do - let(:due_date) { DateTime.strptime('2011-01-28T19:01:12Z') } - let(:raw_data) { data.merge(due_on: due_date) } + context 'when importing a Gitea project' do + let(:iid_attr) { :id } - it 'returns formatted attributes' do - expected = { - iid: 1347, - project: project, - title: '1.0', - description: 'Version 1.0', - state: 'active', - due_date: due_date, - created_at: created_at, - updated_at: updated_at - } - - expect(formatter.attributes).to eq(expected) + before do + project.update!(import_type: 'gitea') end + + it_behaves_like 'Gitlab::LegacyGithubImport::MilestoneFormatter#attributes' end end - context 'when importing a GitHub project' do - it_behaves_like 'Gitlab::LegacyGithubImport::MilestoneFormatter#attributes' + describe '#contributing_user_formatters' do + let(:raw_data) { base_data } + + it { expect(milestone.contributing_user_formatters).to eq({}) } + + it 'includes all user reference columns in #attributes' do + expect(milestone.contributing_user_formatters.keys).to match_array( + milestone.attributes.keys & Gitlab::ImportExport::Base::RelationFactory::USER_REFERENCES.map(&:to_sym) + ) + end end - context 'when importing a Gitea project' do - let(:iid_attr) { :id } - - before do - project.update!(import_type: 'gitea') + describe '#create!', :aggregate_failures, :clean_gitlab_redis_shared_state do + let(:raw_data) { base_data } + let(:store) do + Import::PlaceholderReferences::Store.new( + import_source: ::Import::SOURCE_GITEA, + import_uid: project.import_state.id + ) end - it_behaves_like 'Gitlab::LegacyGithubImport::MilestoneFormatter#attributes' + it 'creates the milestone' do + expect { milestone.create! }.to change { project.milestones.count }.from(0).to(1) + end + + it 'does not push any placeholder references because it does not reference a user' do + milestone_user_refs = milestone.attributes.keys & Gitlab::ImportExport::Base::RelationFactory::USER_REFERENCES + milestone.create! + + expect(store.empty?).to be(true) + expect(milestone_user_refs).to be_empty + end end end diff --git a/spec/lib/gitlab/legacy_github_import/project_creator_spec.rb b/spec/lib/gitlab/legacy_github_import/project_creator_spec.rb index 15624a0558e..04748ee8bdb 100644 --- a/spec/lib/gitlab/legacy_github_import/project_creator_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/project_creator_spec.rb @@ -24,7 +24,7 @@ RSpec.describe Gitlab::LegacyGithubImport::ProjectCreator do allow(project).to receive(:add_import_job) end - stub_application_setting(import_sources: ['github']) + stub_application_setting(import_sources: %w[github gitea]) end describe '#execute' do @@ -40,6 +40,12 @@ RSpec.describe Gitlab::LegacyGithubImport::ProjectCreator do expect(project.import_data.credentials).to eq(user: 'asdffg', password: nil) end + it 'sets user_contribution_mapping_enabled to false' do + project = service.execute + + expect(project.import_data.data["user_contribution_mapping_enabled"]).to eq(false) + end + context 'when GitHub project is private' do it 'sets project visibility to private' do repo[:private] = true @@ -123,5 +129,39 @@ RSpec.describe Gitlab::LegacyGithubImport::ProjectCreator do expect(project.wiki.repository_exists?).to eq true end end + + context 'when the project is imported from Gitea' do + subject(:service) { described_class.new(repo, repo[:name], namespace, user, type: :gitea) } + + it 'sets user_contribution_mapping_enabled to true' do + project = service.execute + + expect(project.import_data.data["user_contribution_mapping_enabled"]).to eq(true) + end + + context 'and gitea_user_mapping is disabled' do + before do + stub_feature_flags(gitea_user_mapping: false) + end + + it 'sets user_contribution_mapping_enabled to false' do + project = service.execute + + expect(project.import_data.data["user_contribution_mapping_enabled"]).to eq(false) + end + end + + context 'and importer_user_mapping is disabled' do + before do + stub_feature_flags(importer_user_mapping: false) + end + + it 'sets user_contribution_mapping_enabled to false' do + project = service.execute + + expect(project.import_data.data["user_contribution_mapping_enabled"]).to eq(false) + end + end + end end end diff --git a/spec/lib/gitlab/legacy_github_import/pull_request_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/pull_request_formatter_spec.rb index 49856c946de..8afa1b4017b 100644 --- a/spec/lib/gitlab/legacy_github_import/pull_request_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/pull_request_formatter_spec.rb @@ -2,9 +2,38 @@ require 'spec_helper' -RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_category: :importers do - let_it_be(:project) { create(:project, :repository, import_type: 'gitea') } - let(:client) { double } +RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, :clean_gitlab_redis_shared_state, feature_category: :importers do + let_it_be(:project) do + create( + :project, + :repository, + :with_import_url, + :import_user_mapping_enabled, + import_type: ::Import::SOURCE_GITEA + ) + end + + let_it_be(:source_user_mapper) do + Gitlab::Import::SourceUserMapper.new( + namespace: project.root_ancestor, + import_type: project.import_type, + source_hostname: 'https://gitea.com' + ) + end + + let_it_be(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } + let_it_be(:import_source_user) do + create( + :import_source_user, + source_user_identifier: octocat[:id], + namespace: project.root_ancestor, + source_hostname: 'https://gitea.com', + import_type: ::Import::SOURCE_GITEA + ) + end + + let(:client) { instance_double(Gitlab::LegacyGithubImport::Client) } + let(:ghost_user) { { id: -1, login: 'Ghost' } } let(:source_sha) { create(:commit, project: project).id } let(:target_commit) { create(:commit, project: project, git_commit: RepoHelpers.another_sample_commit) } let(:target_sha) { target_commit.id } @@ -18,7 +47,6 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor let(:removed_branch) { { ref: 'removed-branch', repo: source_repo, sha: '2e5d3239642f9161dcbbc4b70a211a68e5e45e2b', user: octocat } } let(:forked_branch) { { ref: 'master', repo: forked_source_repo, sha: '2e5d3239642f9161dcbbc4b70a211a68e5e45e2b', user: octocat } } let(:branch_deleted_repo) { { ref: 'master', repo: nil, sha: '2e5d3239642f9161dcbbc4b70a211a68e5e45e2b', user: octocat } } - let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } let(:updated_at) { DateTime.strptime('2011-01-27T19:01:12Z') } let(:imported_from) { ::Import::SOURCE_GITEA } @@ -42,7 +70,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor } end - subject(:pull_request) { described_class.new(project, raw_data, client) } + subject(:pull_request) { described_class.new(project, raw_data, client, source_user_mapper) } before do allow(client).to receive(:user).and_return(octocat) @@ -56,7 +84,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor expected = { iid: 1347, title: 'New feature', - description: "*Created by: octocat*\n\nPlease pull these awesome changes", + description: "Please pull these awesome changes", source_project: project, source_branch: 'branch-merged', source_branch_sha: source_sha, @@ -65,7 +93,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor target_branch_sha: target_sha, state: 'opened', milestone: nil, - author_id: project.creator_id, + author_id: import_source_user.placeholder_user_id, assignee_id: nil, created_at: created_at, updated_at: updated_at, @@ -83,7 +111,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor expected = { iid: 1347, title: 'New feature', - description: "*Created by: octocat*\n\nPlease pull these awesome changes", + description: "Please pull these awesome changes", source_project: project, source_branch: 'branch-merged', source_branch_sha: source_sha, @@ -92,7 +120,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor target_branch_sha: target_sha, state: 'closed', milestone: nil, - author_id: project.creator_id, + author_id: import_source_user.placeholder_user_id, assignee_id: nil, created_at: created_at, updated_at: updated_at, @@ -111,7 +139,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor expected = { iid: 1347, title: 'New feature', - description: "*Created by: octocat*\n\nPlease pull these awesome changes", + description: "Please pull these awesome changes", source_project: project, source_branch: 'branch-merged', source_branch_sha: source_sha, @@ -120,7 +148,7 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor target_branch_sha: target_sha, state: 'merged', milestone: nil, - author_id: project.creator_id, + author_id: import_source_user.placeholder_user_id, assignee_id: nil, created_at: created_at, updated_at: updated_at, @@ -132,36 +160,103 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor end context 'when it is assigned to someone' do - let(:raw_data) { base_data.merge(assignee: octocat) } + context 'and the assigned user has a placeholder user in gitlab' do + let(:raw_data) { base_data.merge(assignee: octocat) } - it 'returns nil as assignee_id when is not a GitLab user' do - expect(pull_request.attributes.fetch(:assignee_id)).to be_nil + it 'returns an existing placeholder user id' do + expect(pull_request.attributes.fetch(:assignee_id)).to eq(import_source_user.placeholder_user_id) + end end - it 'returns GitLab user id associated with GitHub email as assignee_id' do - gl_user = create(:user, email: octocat[:email]) + context 'and the assigned user does not already have a placeholder user' do + let(:octocat_2) { { id: 999999, login: 'octocat two', email: 'octocat2@example.com' } } + let(:raw_data) { base_data.merge(assignee: octocat_2) } - expect(pull_request.attributes.fetch(:assignee_id)).to eq gl_user.id + it 'creates and returns a new placeholder user id', :aggregate_failures do + assignee_id = pull_request.attributes.fetch(:assignee_id) + + expect(User.find(assignee_id).user_type).to eq('placeholder') + expect(assignee_id).not_to eq(import_source_user.placeholder_user_id) + end + end + + context 'and it is assigned to a deleted gitea user' do + let(:raw_data) { base_data.merge(assignee: ghost_user) } + + it 'returns nil for assignee_id' do + expect(pull_request.attributes.fetch(:assignee_id)).to be_nil + end + end + + context 'and user contribution mapping is disabled' do + let(:raw_data) { base_data.merge(assignee: octocat) } + + before do + allow(project).to receive_message_chain(:import_data, :user_mapping_enabled?).and_return(false) + end + + it 'returns nil as assignee_id when is not a GitLab user' do + expect(pull_request.attributes.fetch(:assignee_id)).to be_nil + end + + it 'returns GitLab user id associated with Gitea email as assignee_id' do + gl_user = create(:user, email: octocat[:email]) + + expect(pull_request.attributes.fetch(:assignee_id)).to eq gl_user.id + end end end - context 'when author is a GitLab user' do - let(:raw_data) { base_data.merge(user: octocat) } + context 'when pull request has an author' do + context 'and the author has a placeholder user in gitlab' do + let(:raw_data) { base_data.merge(user: octocat) } - it 'returns project creator_id as author_id when is not a GitLab user' do - expect(pull_request.attributes.fetch(:author_id)).to eq project.creator_id + it 'returns an existing placeholder user id' do + expect(pull_request.attributes.fetch(:author_id)).to eq(import_source_user.placeholder_user_id) + end end - it 'returns GitLab user id associated with GitHub email as author_id' do - gl_user = create(:user, email: octocat[:email]) + context 'and the author does not already have a placeholder user' do + let(:octocat_2) { { id: 999999, login: 'octocat two', email: 'octocat2@example.com' } } + let(:raw_data) { base_data.merge(user: octocat_2) } - expect(pull_request.attributes.fetch(:author_id)).to eq gl_user.id + it 'creates and returns a new placeholder user id', :aggregate_failures do + author_id = pull_request.attributes.fetch(:author_id) + expect(User.find(author_id).user_type).to eq('placeholder') + expect(author_id).not_to eq(import_source_user.placeholder_user_id) + end end - it 'returns description without created at tag line' do - create(:user, email: octocat[:email]) + context 'and the author is a deleted gitea user' do + let(:raw_data) { base_data.merge(user: ghost_user) } - expect(pull_request.attributes.fetch(:description)).to eq('Please pull these awesome changes') + it 'returns the project creator id' do + expect(pull_request.attributes.fetch(:author_id)).to eq(project.creator_id) + end + end + + context 'and user contribution mapping is disabled' do + let(:raw_data) { base_data.merge(user: octocat) } + + before do + allow(project).to receive_message_chain(:import_data, :user_mapping_enabled?).and_return(false) + end + + it 'returns project creator_id as author_id when is not a GitLab user' do + expect(pull_request.attributes.fetch(:author_id)).to eq project.creator_id + end + + it 'returns GitLab user id associated with Gitea email as author_id' do + gl_user = create(:user, email: octocat[:email]) + + expect(pull_request.attributes.fetch(:author_id)).to eq gl_user.id + end + + it 'returns description without created at tag line' do + create(:user, email: octocat[:email]) + + expect(pull_request.attributes.fetch(:description)).to eq('Please pull these awesome changes') + end end end @@ -249,12 +344,36 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor end context 'when importing a GitHub project' do - let(:imported_from) { ::Import::SOURCE_GITHUB } - - before do - project.import_type = 'github' + let_it_be(:project) do + create( + :project, + :repository, + :with_import_url, + :import_user_mapping_enabled, + import_type: ::Import::SOURCE_GITHUB + ) end + let_it_be(:source_user_mapper) do + Gitlab::Import::SourceUserMapper.new( + namespace: project.root_ancestor, + import_type: project.import_type, + source_hostname: 'https://github.com' + ) + end + + let_it_be(:import_source_user) do + create( + :import_source_user, + source_user_identifier: octocat[:id], + namespace: project.root_ancestor, + source_hostname: 'https://github.com', + import_type: ::Import::SOURCE_GITHUB + ) + end + + let(:imported_from) { ::Import::SOURCE_GITHUB } + it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#attributes' it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#number' it_behaves_like 'Gitlab::LegacyGithubImport::PullRequestFormatter#source_branch_name' @@ -338,4 +457,104 @@ RSpec.describe Gitlab::LegacyGithubImport::PullRequestFormatter, feature_categor expect(pull_request.opened?).to be_truthy end end + + describe '#project_association' do + let(:raw_data) { base_data } + + it { expect(pull_request.project_association).to eq(:merge_requests) } + end + + describe '#project_assignee_association' do + let(:raw_data) { base_data } + + it { expect(pull_request.project_assignee_association).to eq(:merge_request_assignees) } + end + + describe '#contributing_user_formatters' do + let(:raw_data) { base_data } + + it 'returns a hash containing UserFormatters for user references in attributes' do + expect(pull_request.contributing_user_formatters).to match( + a_hash_including({ author_id: a_kind_of(Gitlab::LegacyGithubImport::UserFormatter) }) + ) + end + + it 'includes all user reference columns in #attributes' do + all_user_references = Gitlab::ImportExport::Base::RelationFactory::USER_REFERENCES.map(&:to_sym) + + # assignee_id does not need a reference from the attribute on the MR, it's handled through merge_request_assignees + expect(pull_request.contributing_user_formatters.keys).to match_array( + (pull_request.attributes.keys & all_user_references) - [:assignee_id] + ) + end + end + + describe '#contributing_assignee_formatters' do + let(:raw_data) { base_data.merge(assignee: octocat) } + + it 'returns a hash containing the author UserFormatter' do + expect(pull_request.contributing_assignee_formatters).to match( + a_hash_including({ user_id: a_kind_of(Gitlab::LegacyGithubImport::UserFormatter) }) + ) + end + end + + describe '#create!', :aggregate_failures, :clean_gitlab_redis_shared_state do + let(:raw_data) { base_data.merge(assignee: octocat) } + let(:store) do + Import::PlaceholderReferences::Store.new(import_source: imported_from, import_uid: project.import_state.id) + end + + it 'saves the pull_request and assignees' do + pull_request.create! + created_pull_request = project.merge_requests.find_by_iid(pull_request.attributes[:iid]) + + expect(created_pull_request).not_to be_nil + expect(created_pull_request&.merge_request_assignees).not_to be_empty + end + + it 'pushes placeholder references for user references on the pull_request' do + pull_request.create! + cached_references = store.get(100).filter_map do |item| + reference = Import::SourceUserPlaceholderReference.from_serialized(item) + reference if reference.model == 'MergeRequest' + end + + expect(cached_references.map(&:model)).to eq(['MergeRequest']) + expect(cached_references.map(&:source_user_id)).to eq([import_source_user.id]) + expect(cached_references.map(&:user_reference_column)).to eq(['author_id']) + end + + it 'pushes placeholder references for user references on the pull_request assignees' do + pull_request.create! + cached_references = store.get(100).filter_map do |item| + reference = Import::SourceUserPlaceholderReference.from_serialized(item) + reference if reference.model == 'MergeRequestAssignee' + end + + expect(cached_references.map(&:model)).to match_array(['MergeRequestAssignee']) + expect(cached_references.map(&:source_user_id).uniq).to eq([import_source_user.id]) + expect(cached_references.map(&:user_reference_column)).to match_array(['user_id']) + end + + context 'when the pull_request references deleted users in Gitea' do + let(:raw_data) { base_data.merge(user: ghost_user, assignee: ghost_user) } + + it 'does not push any placeholder references' do + pull_request.create! + expect(store.empty?).to eq(true) + end + end + + context 'when user contribution mapping is disabled' do + before do + allow(project).to receive_message_chain(:import_data, :user_mapping_enabled?).and_return(false) + end + + it 'does not push any placeholder references' do + pull_request.create! + expect(store.empty?).to eq(true) + end + end + end end diff --git a/spec/lib/gitlab/legacy_github_import/release_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/release_formatter_spec.rb index 237646f81dc..184e94b0abf 100644 --- a/spec/lib/gitlab/legacy_github_import/release_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/release_formatter_spec.rb @@ -3,7 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::ReleaseFormatter do - let_it_be(:project) { create(:project, namespace: create(:namespace, path: 'octocat')) } + let_it_be(:project) { create(:project, :with_import_url, :import_user_mapping_enabled) } + let(:octocat) { { id: 123456, login: 'octocat' } } let(:created_at) { DateTime.strptime('2011-01-26T19:01:12Z') } let(:published_at) { DateTime.strptime('2011-01-26T20:00:00Z') } diff --git a/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb b/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb index d387d79aa30..b51de92c32a 100644 --- a/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb +++ b/spec/lib/gitlab/legacy_github_import/user_formatter_spec.rb @@ -3,59 +3,178 @@ require 'spec_helper' RSpec.describe Gitlab::LegacyGithubImport::UserFormatter, feature_category: :importers do - let(:client) { double } - let(:octocat) { { id: 123456, login: 'octocat', email: 'octocat@example.com' } } - let(:gitea_ghost) { { id: -1, login: 'Ghost', email: '' } } + let_it_be(:project) { create(:project, :import_user_mapping_enabled, import_type: 'gitea') } + + let_it_be(:source_user_mapper) do + Gitlab::Import::SourceUserMapper.new( + namespace: project.root_ancestor, + import_type: project.import_type, + source_hostname: 'https://gitea.com' + ) + end + + let(:client) { instance_double(Gitlab::LegacyGithubImport::Client) } + let(:gitea_user) { { id: 123456, login: 'octocat', full_name: 'Git Tea', email: 'user@email.com' } } + let(:ghost_user) { { id: -1, login: 'Ghost' } } + + subject(:user_formatter) { described_class.new(client, gitea_user, project, source_user_mapper) } describe '#gitlab_id' do - subject(:user) { described_class.new(client, octocat) } - - before do - allow(client).to receive(:user).and_return(octocat) - end - - context 'when GitHub user is a GitLab user' do - it 'returns GitLab user id when user confirmed primary email matches GitHub email' do - gl_user = create(:user, email: octocat[:email]) - - expect(user.gitlab_id).to eq gl_user.id + context 'when the user exists on Gitea' do + before do + allow(client).to receive(:user).and_return(gitea_user) end - it 'returns GitLab user id when user unconfirmed primary email matches GitHub email' do - gl_user = create(:user, :unconfirmed, email: octocat[:email]) + context 'when a placeholder user does not exist for the id from Gitea' do + it 'creates a new source user' do + expect { user_formatter.gitlab_id }.to change { Import::SourceUser.count }.from(0).to(1) + end - expect(user.gitlab_id).to eq gl_user.id + it 'returns a new placeholder user id' do + expect(user_formatter.gitlab_id).not_to be_nil + expect(User.find(user_formatter.gitlab_id)).to be_placeholder + end end - it 'returns GitLab user id when user confirmed secondary email matches GitHub email' do - gl_user = create(:user, email: 'johndoe@example.com') - create(:email, :confirmed, user: gl_user, email: octocat[:email]) + context 'when a placeholder user exists for the id from Gitea' do + let!(:source_user) do + create( + :import_source_user, + source_user_identifier: gitea_user[:id], + source_hostname: 'https://gitea.com', + import_type: project.import_type, + namespace: project.root_ancestor + ) + end - expect(user.gitlab_id).to eq gl_user.id + it 'returns the existing placeholder user id' do + expect(user_formatter.gitlab_id).to eq(source_user.placeholder_user_id) + end end - it 'returns nil when user unconfirmed secondary email matches GitHub email' do - gl_user = create(:user, email: 'johndoe@example.com') - create(:email, user: gl_user, email: octocat[:email]) + context 'when a placeholder has already been reassigned to a real user' do + let!(:source_user) do + create( + :import_source_user, + :completed, + source_user_identifier: gitea_user[:id], + source_hostname: 'https://gitea.com', + import_type: project.import_type, + namespace: project.root_ancestor + ) + end - expect(user.gitlab_id).to be_nil + it 'returns the reassigned user id' do + expect(user_formatter.gitlab_id).to eq(source_user.reassign_to_user_id) + end + end + + context 'when user contribution mapping is disabled' do + before do + allow(client).to receive(:user).and_return(gitea_user) + allow(project).to receive_message_chain(:import_data, :user_mapping_enabled?).and_return(false) + end + + it 'returns GitLab user id when user confirmed primary email matches Gitea email' do + gl_user = create(:user, email: gitea_user[:email]) + + expect(user_formatter.gitlab_id).to eq gl_user.id + end + + it 'returns GitLab user id when user unconfirmed primary email matches Gitea email' do + gl_user = create(:user, :unconfirmed, email: gitea_user[:email]) + + expect(user_formatter.gitlab_id).to eq gl_user.id + end + + it 'returns GitLab user id when user confirmed secondary email matches Gitea email' do + gl_user = create(:user, email: 'johndoe@example.com') + create(:email, :confirmed, user: gl_user, email: gitea_user[:email]) + + expect(user_formatter.gitlab_id).to eq gl_user.id + end + + it 'returns nil when user unconfirmed secondary email matches Gitea email' do + gl_user = create(:user, email: 'johndoe@example.com') + create(:email, user: gl_user, email: gitea_user[:email]) + + expect(user_formatter.gitlab_id).to be_nil + end end end - it 'returns nil when GitHub user is not a GitLab user' do - expect(user.gitlab_id).to be_nil + context 'when the user has been deleted on Gitea' do + subject(:user_formatter) { described_class.new(client, ghost_user, project, source_user_mapper) } + + it 'returns nil' do + expect(user_formatter.gitlab_id).to be_nil + end + + it 'does not create a placeholder user for ghost users' do + expect { user_formatter.gitlab_id }.not_to change { Import::SourceUser.count }.from(0) + expect { user_formatter.gitlab_id }.not_to change { User.where(user_type: :placeholder).count }.from(0) + end + + context 'and improved user mapping is disabled' do + before do + allow(client).to receive(:user).and_return(ghost_user) + allow(project).to receive_message_chain(:import_data, :user_mapping_enabled?).and_return(false) + end + + it 'returns nil' do + expect(user_formatter.gitlab_id).to be_nil + end + end end end - describe '.email' do - subject(:user) { described_class.new(client, gitea_ghost) } + describe '#source_user', :aggregate_failures do + context 'when the user exists on Gitea' do + before do + allow(client).to receive(:user).and_return(gitea_user) + end - before do - allow(client).to receive(:user).and_return(gitea_ghost) + context 'and a source user does not exist' do + it 'creates and returns new source user' do + expect { user_formatter.source_user }.to change { Import::SourceUser.count }.from(0).to(1) + expect(user_formatter.source_user.class).to eq(Import::SourceUser) + end + end + + context 'and a source user already exists' do + let!(:source_user) do + create( + :import_source_user, + source_user_identifier: gitea_user[:id], + source_hostname: 'https://gitea.com', + import_type: project.import_type, + namespace: project.root_ancestor + ) + end + + it 'returns the existing source user' do + expect(user_formatter.source_user.id).to eq(source_user.id) + end + end end - it 'assigns a dummy email address when user is a Ghost gitea user' do - expect(subject.send(:email)).to eq described_class::GITEA_GHOST_EMAIL + context 'when the source user has been deleted on gitea' do + subject(:user_formatter) { described_class.new(client, ghost_user, project, source_user_mapper) } + + it 'returns nil' do + expect(user_formatter.source_user).to be_nil + end + end + + context 'when user contribution mapping is disabled' do + before do + allow(client).to receive(:user).and_return(gitea_user) + allow(project).to receive_message_chain(:import_data, :user_mapping_enabled?).and_return(false) + end + + it 'returns nil' do + expect(user_formatter.source_user).to be_nil + end end end end diff --git a/spec/models/project_import_data_spec.rb b/spec/models/project_import_data_spec.rb index 50a2ee42084..b83763a4ad6 100644 --- a/spec/models/project_import_data_spec.rb +++ b/spec/models/project_import_data_spec.rb @@ -50,4 +50,24 @@ RSpec.describe ProjectImportData do expect(row.credentials).to eq({}) end end + + describe '#user_mapping_enabled?' do + it 'returns user_contribution_mapping_enabled when present in data' do + import_data = described_class.new(data: { 'user_contribution_mapping_enabled' => true }) + + expect(import_data.user_mapping_enabled?).to be(true) + end + + it 'returns false when user_contribution_mapping_enabled is not present in data' do + import_data = described_class.new(data: { 'number' => 10 }) + + expect(import_data.user_mapping_enabled?).to be(false) + end + + it 'returns false when data is nil' do + import_data = described_class.new + + expect(import_data.user_mapping_enabled?).to be(false) + end + end end diff --git a/workhorse/go.mod b/workhorse/go.mod index 01ee3470f95..775cac30965 100644 --- a/workhorse/go.mod +++ b/workhorse/go.mod @@ -13,7 +13,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.27.42 github.com/aws/aws-sdk-go-v2/credentials v1.17.41 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.25 - github.com/aws/aws-sdk-go-v2/service/s3 v1.63.3 + github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0 github.com/disintegration/imaging v1.6.2 github.com/dlclark/regexp2 v1.11.4 github.com/getsentry/raven-go v0.2.0 @@ -60,16 +60,16 @@ require ( github.com/DataDog/datadog-go v4.4.0+incompatible // indirect github.com/DataDog/sketches-go v1.0.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect diff --git a/workhorse/go.sum b/workhorse/go.sum index 424c2fb909e..5a497f3afa8 100644 --- a/workhorse/go.sum +++ b/workhorse/go.sum @@ -106,8 +106,8 @@ github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 h1:xDAuZTn4IMm8o1LnBZvmrL8JA1io4o3YWNXgohbf20g= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5/go.mod h1:wYSv6iDS621sEFLfKvpPE2ugjTuGlAG7iROg0hLOkfc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= github.com/aws/aws-sdk-go-v2/config v1.27.42 h1:Zsy9coUPuOsCWkjTvHpl2/DB9bptXtv7WeNPxvFr87s= github.com/aws/aws-sdk-go-v2/config v1.27.42/go.mod h1:FGASs+PuJM2EY+8rt8qyQKLPbbX/S5oY+6WzJ/KE7ko= github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8= @@ -122,18 +122,18 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYE github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 h1:OWYvKL53l1rbsUmW7bQyJVsYU/Ii3bbAAQIIFNbM0Tk= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18/go.mod h1:CUx0G1v3wG6l01tUB+j7Y8kclA8NSqK4ef0YG79a4cg= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 h1:7edmS3VOBDhK00b/MwGtGglCm7hhwNYnjJs/PgFdMQE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21/go.mod h1:Q9o5h4HoIWG8XfzxqiuK/CGUbepCJ8uTlaE3bAbxytQ= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 h1:rTWjG6AvWekO2B1LHeM3ktU7MqyX9rzWQ7hgzneZW7E= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20/go.mod h1:RGW2DDpVc8hu6Y6yG8G5CHVmVOAn1oV8rNKOHRJyswg= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 h1:4FMHqLfk0efmTqhXVRL5xYRqlEBNBiRI7N6w4jsEdd4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2/go.mod h1:LWoqeWlK9OZeJxsROW2RqrSPvQHKTpp69r/iDjwsSaw= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 h1:eb+tFOIl9ZsUe2259/BKPeniKuz4/02zZFH/i4Nf8Rg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18/go.mod h1:GVCC2IJNJTmdlyEsSmofEy7EfJncP7DNnXDzRjJ5Keg= -github.com/aws/aws-sdk-go-v2/service/s3 v1.63.3 h1:3zt8qqznMuAZWDTDpcwv9Xr11M/lVj2FsRR7oYBt0OA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.63.3/go.mod h1:NLTqRLe3pUNu3nTEHI6XlHLKYmc8fbHUdMxAB6+s41Q= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt3E41huP+GvQZJD38WLsgVp4iOtAjg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2/go.mod h1:/niFCtmuQNxqx9v8WAPq5qh7EH25U4BF6tjoyq9bObM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0 h1:xA6XhTF7PE89BCNHJbQi8VvPzcgMtmGC5dr8S8N7lHk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk= github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o=