111 lines
3.1 KiB
Ruby
111 lines
3.1 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
module Gitlab
|
|
module GithubImport
|
|
module BulkImporting
|
|
attr_reader :project, :client
|
|
|
|
# project - An instance of `Project`.
|
|
# client - An instance of `Gitlab::GithubImport::Client`.
|
|
def initialize(project, client)
|
|
@project = project
|
|
@client = client
|
|
@validation_errors = []
|
|
end
|
|
|
|
# Builds and returns an Array of objects to bulk insert into the
|
|
# database and array of validation errors if object is invalid.
|
|
#
|
|
# enum - An Enumerable that returns the objects to turn into database
|
|
# rows.
|
|
def build_database_rows(enum)
|
|
errors = []
|
|
rows = enum.each_with_object([]) do |(object, _), result|
|
|
next if already_imported?(object)
|
|
|
|
attrs = build_attributes(object)
|
|
build_record = model.new(attrs)
|
|
|
|
if build_record.invalid?
|
|
github_identifiers = github_identifiers(object)
|
|
|
|
log_error(github_identifiers, build_record.errors.full_messages)
|
|
errors << {
|
|
validation_errors: build_record.errors,
|
|
external_identifiers: github_identifiers
|
|
}
|
|
next
|
|
end
|
|
|
|
result << attrs
|
|
end
|
|
|
|
log_and_increment_counter(rows.size, :fetched)
|
|
|
|
[rows, errors]
|
|
end
|
|
|
|
# Bulk inserts the given rows into the database.
|
|
def bulk_insert(rows, batch_size: 100)
|
|
rows.each_slice(batch_size) do |slice|
|
|
ApplicationRecord.legacy_bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert
|
|
|
|
log_and_increment_counter(slice.size, :imported)
|
|
end
|
|
end
|
|
|
|
def object_type
|
|
raise NotImplementedError
|
|
end
|
|
|
|
def bulk_insert_failures(errors)
|
|
rows = errors.map do |error|
|
|
correlation_id_value = Labkit::Correlation::CorrelationId.current_or_new_id
|
|
|
|
{
|
|
source: self.class.name,
|
|
exception_class: 'ActiveRecord::RecordInvalid',
|
|
exception_message: error[:validation_errors].full_messages.first.truncate(255),
|
|
correlation_id_value: correlation_id_value,
|
|
retry_count: nil,
|
|
created_at: Time.zone.now,
|
|
external_identifiers: error[:external_identifiers]
|
|
}
|
|
end
|
|
|
|
project.import_failures.insert_all(rows)
|
|
end
|
|
|
|
private
|
|
|
|
def log_and_increment_counter(value, operation)
|
|
Logger.info(
|
|
project_id: project.id,
|
|
importer: self.class.name,
|
|
message: "#{value} #{object_type.to_s.pluralize} #{operation}"
|
|
)
|
|
|
|
Gitlab::GithubImport::ObjectCounter.increment(
|
|
project,
|
|
object_type,
|
|
operation,
|
|
value: value
|
|
)
|
|
end
|
|
|
|
def log_error(github_identifiers, messages)
|
|
Logger.error(
|
|
project_id: project.id,
|
|
importer: self.class.name,
|
|
message: messages,
|
|
external_identifiers: github_identifiers
|
|
)
|
|
end
|
|
|
|
def github_identifiers(object)
|
|
raise NotImplementedError
|
|
end
|
|
end
|
|
end
|
|
end
|