root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Move code to a BackgroundMigration 

And run in intervals.
This commit is contained in:
Toon Claes 2018-10-16 13:39:18 +02:00
parent 1f2c9915f6
commit cc70bd8440
3 changed files with 164 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
db/post_migrate/20181010133639_backfill_store_project_full_path_in_repo.rb
View File

@ -3,134 +3,25 @@
class BackfillStoreProjectFullPathInRepo < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
DOWNTIME = false
BATCH_SIZE = 1_000
DELAY_INTERVAL = 5.minutes
UP_MIGRATION = 'BackfillProjectFullpathInRepoConfig::Up'
DOWN_MIGRATION = 'BackfillProjectFullpathInRepoConfig::Down'
class Repository
attr_reader :storage
def initialize(storage, relative_path)
@storage = storage
@relative_path = relative_path
end
def gitaly_repository
Gitaly::Repository.new(storage_name: @storage, relative_path: @relative_path)
end
end
module Storage
class HashedProject
attr_accessor :project
ROOT_PATH_PREFIX = '@hashed'.freeze
def initialize(project)
@project = project
end
def disk_path
"#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}"
end
def disk_hash
@disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id
end
end
class LegacyProject
attr_accessor :project
def initialize(project)
@project = project
end
def disk_path
project.full_path
end
end
end
module Routable
extend ActiveSupport::Concern
def full_path
@full_path ||= build_full_path
end
def build_full_path
if parent && path
parent.full_path + '/' + path
else
path
end
end
end
class Namespace < ActiveRecord::Base
self.table_name = 'namespaces'
include Routable
belongs_to :parent, class_name: "Namespace"
end
disable_ddl_transaction!
class Project < ActiveRecord::Base
self.table_name = 'projects'
include Routable
include EachBatch
FULLPATH_CONFIG_KEY = 'gitlab.fullpath'
belongs_to :namespace
delegate :disk_path, to: :storage
alias_method :parent, :namespace
def add_fullpath_config
entries = { FULLPATH_CONFIG_KEY => full_path }
repository_service.set_config(entries)
end
def remove_fullpath_config
repository_service.delete_config([FULLPATH_CONFIG_KEY])
end
def storage
@storage ||=
if hashed_storage?
Storage::HashedProject.new(self)
else
Storage::LegacyProject.new(self)
end
end
def hashed_storage?
self.storage_version && self.storage_version >= 1
end
def repository
@repository ||= Repository.new(repository_storage, disk_path + '.git')
end
def repository_service
@repository_service ||= Gitlab::GitalyClient::RepositoryService.new(repository)
end
end
def up
Project.each_batch do |batch|
batch.each do |project|
project.add_fullpath_config
end
end
queue_background_migration_jobs_by_range_at_intervals(Project, UP_MIGRATION, DELAY_INTERVAL)
end
def down
Project.each_batch do |batch|
batch.each do |project|
project.remove_fullpath_config
end
end
queue_background_migration_jobs_by_range_at_intervals(Project, DOWN_MIGRATION, DELAY_INTERVAL)
end
end

150
lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb Normal file
View File

@ -0,0 +1,150 @@
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This module is used to write the full path of all projects to
# the git repository config file.
# Storing the full project path in the git config allows admins to
# easily identify a project when it is using hashed storage.
module BackfillProjectFullpathInRepoConfig
module Storage
# Class that returns the disk path for a project using hashed storage
class HashedProject
attr_accessor :project
ROOT_PATH_PREFIX = '@hashed'.freeze
def initialize(project)
@project = project
end
def disk_path
"#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}"
end
def disk_hash
@disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id
end
end
# Class that returns the disk path for a project using legacy storage
class LegacyProject
attr_accessor :project
def initialize(project)
@project = project
end
def disk_path
project.full_path
end
end
end
# Concern used by Project and Namespace to determine the full
# route the the project
module Routable
extend ActiveSupport::Concern
def full_path
@full_path ||= build_full_path
end
def build_full_path
if parent && path
parent.full_path + '/' + path
else
path
end
end
end
# Class used to interact with repository using Gitaly
class Repository
attr_reader :storage
def initialize(storage, relative_path)
@storage = storage
@relative_path = relative_path
end
def gitaly_repository
Gitaly::Repository.new(storage_name: @storage, relative_path: @relative_path)
end
end
# Namespace can be a user or group. It can be the root or a
# child of another namespace.
class Namespace < ActiveRecord::Base
self.table_name = 'namespaces'
include Routable
belongs_to :parent, class_name: "Namespace"
end
# Project is where the repository (etc.) is stored
class Project < ActiveRecord::Base
self.table_name = 'projects'
include Routable
include EachBatch
FULLPATH_CONFIG_KEY = 'gitlab.fullpath'
belongs_to :namespace
delegate :disk_path, to: :storage
alias_method :parent, :namespace
def add_fullpath_config
entries = { FULLPATH_CONFIG_KEY => full_path }
repository_service.set_config(entries)
end
def remove_fullpath_config
repository_service.delete_config([FULLPATH_CONFIG_KEY])
end
def storage
@storage ||=
if hashed_storage?
Storage::HashedProject.new(self)
else
Storage::LegacyProject.new(self)
end
end
def hashed_storage?
self.storage_version && self.storage_version >= 1
end
def repository
@repository ||= Repository.new(repository_storage, disk_path + '.git')
end
def repository_service
@repository_service ||= Gitlab::GitalyClient::RepositoryService.new(repository)
end
end
# Class to add the fullpath to the git repo config
class Up
def perform(start_id, end_id)
Project.where(id: start_id..end_id).each do |project|
project.add_fullpath_config
end
end
end
# Class to rollback adding the fullpath to the git repo config
class Down
def perform(start_id, end_id)
Project.where(id: start_id..end_id).each do |project|
project.remove_fullpath_config
end
end
end
end
end
end

6
spec/migrations/backfill_store_project_full_path_in_repo_spec.rb
View File

@ -12,6 +12,12 @@ describe BackfillStoreProjectFullPathInRepo, :migration do
subject(:migration) { described_class.new }
around do |example|
Sidekiq::Testing.inline! do
example.run
end
end
describe '#up' do
shared_examples_for 'writes the full path to git config' do
it 'writes the git config' do