root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add latest changes from gitlab-org/security/gitlab@16-1-stable-ee

This commit is contained in:
GitLab Bot 2023-07-31 14:35:52 +00:00
parent 62ed09f455
commit 1e8d7c8554
34 changed files with 592 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/services/bulk_imports/archive_extraction_service.rb
View File

@ -49,11 +49,7 @@ module BulkImports
end
def validate_symlink
raise(BulkImports::Error, 'Invalid file') if symlink?(filepath)
end
def symlink?(filepath)
File.lstat(filepath).symlink?
raise(BulkImports::Error, 'Invalid file') if Gitlab::Utils::FileInfo.linked?(filepath)
end
def extract_archive

2
app/services/bulk_imports/file_decompression_service.rb
View File

@ -53,7 +53,7 @@ module BulkImports
end
def validate_symlink(filepath)
raise(ServiceError, 'Invalid file') if File.lstat(filepath).symlink?
raise(ServiceError, 'Invalid file') if Gitlab::Utils::FileInfo.linked?(filepath)
end
def decompress_file

2
config/application.rb
View File

@ -172,6 +172,7 @@ module Gitlab
# - Any parameter containing `password`
# - Any parameter containing `secret`
# - Any parameter ending with `key`
# - Any parameter named `redirect`, filtered for security concerns of exposing sensitive information
# - Two-factor tokens (:otp_attempt)
# - Repo/Project Import URLs (:import_url)
# - Build traces (:trace)
@ -214,6 +215,7 @@ module Gitlab
variables
content
sharedSecret
redirect
)
# Enable escaping HTML in JSON.

11
lib/banzai/filter/plantuml_filter.rb
View File

@ -11,7 +11,7 @@ module Banzai
def call
return doc unless settings.plantuml_enabled? && doc.at_xpath(lang_tag)
plantuml_setup
Gitlab::Plantuml.configure
doc.xpath(lang_tag).each do |node|
img_tag = Nokogiri::HTML::DocumentFragment.parse(
@ -38,15 +38,6 @@ module Banzai
def settings
Gitlab::CurrentSettings.current_application_settings
end
def plantuml_setup
Asciidoctor::PlantUml.configure do |conf|
conf.url = settings.plantuml_url
conf.png_enable = settings.plantuml_enabled
conf.svg_enable = false
conf.txt_enable = false
end
end
end
end
end

2
lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb
View File

@ -24,7 +24,7 @@ module BulkImports
return if tar_filepath?(file_path)
return if lfs_json_filepath?(file_path)
return if File.directory?(file_path)
return if File.lstat(file_path).symlink?
return if Gitlab::Utils::FileInfo.linked?(file_path)
size = File.size(file_path)
oid = LfsObject.calculate_oid(file_path)

2
lib/bulk_imports/common/pipelines/uploads_pipeline.rb
View File

@ -24,7 +24,7 @@ module BulkImports
# Validate that the path is OK to load
Gitlab::PathTraversal.check_allowed_absolute_path_and_path_traversal!(file_path, [Dir.tmpdir])
return if File.directory?(file_path)
return if File.lstat(file_path).symlink?
return if Gitlab::Utils::FileInfo.linked?(file_path)
avatar_path = AVATAR_PATTERN.match(file_path)
return save_avatar(file_path) if avatar_path

2
lib/bulk_imports/file_downloads/validations.rb
View File

@ -32,7 +32,7 @@ module BulkImports
end
def validate_symlink
return unless File.lstat(filepath).symlink?
return unless Gitlab::Utils::FileInfo.linked?(filepath)
File.delete(filepath)
raise_error 'Invalid downloaded file'

2
lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb
View File

@ -26,7 +26,7 @@ module BulkImports
return unless portable.lfs_enabled?
return unless File.exist?(bundle_path)
return if File.directory?(bundle_path)
return if File.lstat(bundle_path).symlink?
return if Gitlab::Utils::FileInfo.linked?(bundle_path)
portable.design_repository.create_from_bundle(bundle_path)
end

2
lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb
View File

@ -26,7 +26,7 @@ module BulkImports
return unless File.exist?(bundle_path)
return if File.directory?(bundle_path)
return if File.lstat(bundle_path).symlink?
return if Gitlab::Utils::FileInfo.linked?(bundle_path)
portable.repository.create_from_bundle(bundle_path)
end

11
lib/gitlab/asciidoc.rb
View File

@ -78,20 +78,11 @@ module Gitlab
context[:pipeline] = :ascii_doc
context[:max_includes] = [MAX_INCLUDES, context[:max_includes]].compact.min
plantuml_setup
Gitlab::Plantuml.configure
html = ::Asciidoctor.convert(input, asciidoc_opts)
html = Banzai.render(html, context)
html.html_safe
end
def self.plantuml_setup
Asciidoctor::PlantUml.configure do |conf|
conf.url = Gitlab::CurrentSettings.plantuml_url
conf.svg_enable = Gitlab::CurrentSettings.plantuml_enabled
conf.png_enable = Gitlab::CurrentSettings.plantuml_enabled
conf.txt_enable = false
end
end
end
end

2
lib/gitlab/ci/decompressed_gzip_size_validator.rb
View File

@ -65,7 +65,7 @@ module Gitlab
def validate_archive_path
Gitlab::PathTraversal.check_path_traversal!(archive_path)
raise(ServiceError, 'Archive path is a symlink') if File.lstat(archive_path).symlink?
raise(ServiceError, 'Archive path is a symlink or hard link') if Gitlab::Utils::FileInfo.linked?(archive_path)
raise(ServiceError, 'Archive path is not a file') unless File.file?(archive_path)
end

33
lib/gitlab/import_export/command_line_util.rb
View File

@ -5,8 +5,11 @@ module Gitlab
module CommandLineUtil
UNTAR_MASK = 'u+rwX,go+rX,go-w'
DEFAULT_DIR_MODE = 0700
CLEAN_DIR_IGNORE_FILE_NAMES = %w[. ..].freeze
FileOversizedError = Class.new(StandardError)
CommandLineUtilError = Class.new(StandardError)
FileOversizedError = Class.new(CommandLineUtilError)
HardLinkError = Class.new(CommandLineUtilError)
def tar_czf(archive:, dir:)
tar_with_options(archive: archive, dir: dir, options: 'czf')
@ -90,7 +93,7 @@ module Gitlab
def untar_with_options(archive:, dir:, options:)
execute_cmd(%W(tar -#{options} #{archive} -C #{dir}))
execute_cmd(%W(chmod -R #{UNTAR_MASK} #{dir}))
remove_symlinks(dir)
clean_extraction_dir!(dir)
end
# rubocop:disable Gitlab/ModuleWithInstanceVariables
@ -122,17 +125,27 @@ module Gitlab
true
end
def remove_symlinks(dir)
ignore_file_names = %w[. ..]
# Scans and cleans the directory tree.
# Symlinks are considered legal but are removed.
# Files sharing hard links are considered illegal and the directory will be removed
# and a `HardLinkError` exception will be raised.
#
# @raise [HardLinkError] if there multiple hard links to the same file detected.
# @return [Boolean] true
def clean_extraction_dir!(dir)
# Using File::FNM_DOTMATCH to also delete symlinks starting with "."
Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH)
.reject { |f| ignore_file_names.include?(File.basename(f)) }
.each do |filepath|
FileUtils.rm(filepath) if File.lstat(filepath).symlink?
end
Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH).each do |filepath|
next if CLEAN_DIR_IGNORE_FILE_NAMES.include?(File.basename(filepath))
raise HardLinkError, 'File shares hard link' if Gitlab::Utils::FileInfo.shares_hard_link?(filepath)
FileUtils.rm(filepath) if Gitlab::Utils::FileInfo.linked?(filepath)
end
true
rescue HardLinkError
FileUtils.remove_dir(dir)
raise
end
end
end

2
lib/gitlab/import_export/decompressed_archive_size_validator.rb
View File

@ -87,7 +87,7 @@ module Gitlab
def validate_archive_path
Gitlab::PathTraversal.check_path_traversal!(@archive_path)
raise(ServiceError, 'Archive path is a symlink') if File.lstat(@archive_path).symlink?
raise(ServiceError, 'Archive path is a symlink or hard link') if Gitlab::Utils::FileInfo.linked?(@archive_path)
raise(ServiceError, 'Archive path is not a file') unless File.file?(@archive_path)
end

4
lib/gitlab/import_export/file_importer.rb
View File

@ -23,7 +23,7 @@ module Gitlab
mkdir_p(@shared.export_path)
mkdir_p(@shared.archive_path)
remove_symlinks(@shared.export_path)
clean_extraction_dir!(@shared.export_path)
copy_archive
wait_for_archived_file do
@ -35,7 +35,7 @@ module Gitlab
false
ensure
remove_import_file
remove_symlinks(@shared.export_path)
clean_extraction_dir!(@shared.export_path)
end
private

6
lib/gitlab/import_export/json/ndjson_reader.rb
View File

@ -21,7 +21,9 @@ module Gitlab
# This reads from `tree/project.json`
path = file_path("#{importable_path}.json")
raise Gitlab::ImportExport::Error, 'Invalid file' if !File.exist?(path) || File.symlink?(path)
if !File.exist?(path) || Gitlab::Utils::FileInfo.linked?(path)
raise Gitlab::ImportExport::Error, 'Invalid file'
end
data = File.read(path, MAX_JSON_DOCUMENT_SIZE)
json_decode(data)
@ -34,7 +36,7 @@ module Gitlab
# This reads from `tree/project/merge_requests.ndjson`
path = file_path(importable_path, "#{key}.ndjson")
next if !File.exist?(path) || File.symlink?(path)
next if !File.exist?(path) || Gitlab::Utils::FileInfo.linked?(path)
File.foreach(path, MAX_JSON_DOCUMENT_SIZE).with_index do |line, line_num|
documents << [json_decode(line), line_num]

2
lib/gitlab/import_export/recursive_merge_folders.rb
View File

@ -57,7 +57,7 @@ module Gitlab
source_child = File.join(source_path, child)
target_child = File.join(target_path, child)
next if File.lstat(source_child).symlink?
next if Gitlab::Utils::FileInfo.linked?(source_child)
if File.directory?(source_child)
FileUtils.mkdir_p(target_child, mode: DEFAULT_DIR_MODE) unless File.exist?(target_child)

20
lib/gitlab/plantuml.rb Normal file
View File

@ -0,0 +1,20 @@
# frozen_string_literal: true
require "asciidoctor_plantuml/plantuml"
module Gitlab
module Plantuml
class << self
def configure
Asciidoctor::PlantUml.configure do |conf|
conf.url = Gitlab::CurrentSettings.plantuml_url
conf.png_enable = Gitlab::CurrentSettings.plantuml_enabled
conf.svg_enable = false
conf.txt_enable = false
conf
end
end
end
end
end

35
lib/gitlab/utils/file_info.rb Normal file
View File

@ -0,0 +1,35 @@
# frozen_string_literal: true
module Gitlab
module Utils
module FileInfo
class << self
# Returns true if:
# - File or directory is a symlink.
# - File shares a hard link.
def linked?(file)
stat = to_file_stat(file)
stat.symlink? || shares_hard_link?(stat)
end
# Returns:
# - true if file shares a hard link with another file.
# - false if file is a directory, as directories cannot be hard linked.
def shares_hard_link?(file)
stat = to_file_stat(file)
stat.file? && stat.nlink > 1
end
private
def to_file_stat(filepath_or_stat)
return filepath_or_stat if filepath_or_stat.is_a?(File::Stat)
File.lstat(filepath_or_stat)
end
end
end
end
end

15
spec/lib/bulk_imports/common/pipelines/lfs_objects_pipeline_spec.rb
View File

@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe BulkImports::Common::Pipelines::LfsObjectsPipeline do
RSpec.describe BulkImports::Common::Pipelines::LfsObjectsPipeline, feature_category: :importers do
let_it_be(:portable) { create(:project) }
let_it_be(:oid) { 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' }
@ -118,13 +118,22 @@ RSpec.describe BulkImports::Common::Pipelines::LfsObjectsPipeline do
context 'when file path is symlink' do
it 'returns' do
symlink = File.join(tmpdir, 'symlink')
FileUtils.ln_s(lfs_file_path, symlink)
FileUtils.ln_s(File.join(tmpdir, lfs_file_path), symlink)
expect(Gitlab::Utils::FileInfo).to receive(:linked?).with(symlink).and_call_original
expect { pipeline.load(context, symlink) }.not_to change { portable.lfs_objects.count }
end
end
context 'when file path shares multiple hard links' do
it 'returns' do
FileUtils.link(lfs_file_path, File.join(tmpdir, 'hard_link'))
expect(Gitlab::Utils::FileInfo).to receive(:linked?).with(lfs_file_path).and_call_original
expect { pipeline.load(context, lfs_file_path) }.not_to change { portable.lfs_objects.count }
end
end
context 'when path is a directory' do
it 'returns' do
expect { pipeline.load(context, Dir.tmpdir) }.not_to change { portable.lfs_objects.count }

14
spec/lib/bulk_imports/common/pipelines/uploads_pipeline_spec.rb
View File

@ -105,6 +105,7 @@ RSpec.describe BulkImports::Common::Pipelines::UploadsPipeline, feature_category
it 'returns' do
path = File.join(tmpdir, 'test')
FileUtils.touch(path)
expect { pipeline.load(context, path) }.not_to change { portable.uploads.count }
end
end
@ -118,13 +119,22 @@ RSpec.describe BulkImports::Common::Pipelines::UploadsPipeline, feature_category
context 'when path is a symlink' do
it 'does not upload the file' do
symlink = File.join(tmpdir, 'symlink')
FileUtils.ln_s(upload_file_path, symlink)
FileUtils.ln_s(File.join(tmpdir, upload_file_path), symlink)
expect(Gitlab::Utils::FileInfo).to receive(:linked?).with(symlink).and_call_original
expect { pipeline.load(context, symlink) }.not_to change { portable.uploads.count }
end
end
context 'when path has multiple hard links' do
it 'does not upload the file' do
FileUtils.link(upload_file_path, File.join(tmpdir, 'hard_link'))
expect(Gitlab::Utils::FileInfo).to receive(:linked?).with(upload_file_path).and_call_original
expect { pipeline.load(context, upload_file_path) }.not_to change { portable.uploads.count }
end
end
context 'when path traverses' do
it 'does not upload the file' do
path_traversal = "#{uploads_dir_path}/avatar/../../../../etc/passwd"

19
spec/lib/bulk_imports/projects/pipelines/design_bundle_pipeline_spec.rb
View File

@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe BulkImports::Projects::Pipelines::DesignBundlePipeline do
RSpec.describe BulkImports::Projects::Pipelines::DesignBundlePipeline, feature_category: :importers do
let_it_be(:design) { create(:design, :with_file) }
let(:portable) { create(:project) }
@ -125,9 +125,9 @@ RSpec.describe BulkImports::Projects::Pipelines::DesignBundlePipeline do
context 'when path is symlink' do
it 'returns' do
symlink = File.join(tmpdir, 'symlink')
FileUtils.ln_s(design_bundle_path, symlink)
FileUtils.ln_s(File.join(tmpdir, design_bundle_path), symlink)
expect(Gitlab::Utils::FileInfo).to receive(:linked?).with(symlink).and_call_original
expect(portable.design_repository).not_to receive(:create_from_bundle)
pipeline.load(context, symlink)
@ -136,6 +136,19 @@ RSpec.describe BulkImports::Projects::Pipelines::DesignBundlePipeline do
end
end
context 'when path has multiple hard links' do
it 'returns' do
FileUtils.link(design_bundle_path, File.join(tmpdir, 'hard_link'))
expect(Gitlab::Utils::FileInfo).to receive(:linked?).with(design_bundle_path).and_call_original
expect(portable.design_repository).not_to receive(:create_from_bundle)
pipeline.load(context, design_bundle_path)
expect(portable.design_repository.exists?).to eq(false)
end
end
context 'when path is not under tmpdir' do
it 'returns' do
expect { pipeline.load(context, '/home/test.txt') }

19
spec/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline_spec.rb
View File

@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe BulkImports::Projects::Pipelines::RepositoryBundlePipeline do
RSpec.describe BulkImports::Projects::Pipelines::RepositoryBundlePipeline, feature_category: :importers do
let_it_be(:source) { create(:project, :repository) }
let(:portable) { create(:project) }
@ -123,9 +123,9 @@ RSpec.describe BulkImports::Projects::Pipelines::RepositoryBundlePipeline do
context 'when path is symlink' do
it 'returns' do
symlink = File.join(tmpdir, 'symlink')
FileUtils.ln_s(bundle_path, symlink)
FileUtils.ln_s(File.join(tmpdir, bundle_path), symlink)
expect(Gitlab::Utils::FileInfo).to receive(:linked?).with(symlink).and_call_original
expect(portable.repository).not_to receive(:create_from_bundle)
pipeline.load(context, symlink)
@ -134,6 +134,19 @@ RSpec.describe BulkImports::Projects::Pipelines::RepositoryBundlePipeline do
end
end
context 'when path has mutiple hard links' do
it 'returns' do
FileUtils.link(bundle_path, File.join(tmpdir, 'hard_link'))
expect(Gitlab::Utils::FileInfo).to receive(:linked?).with(bundle_path).and_call_original
expect(portable.repository).not_to receive(:create_from_bundle)
pipeline.load(context, bundle_path)
expect(portable.repository.exists?).to eq(false)
end
end
context 'when path is not under tmpdir' do
it 'returns' do
expect { pipeline.load(context, '/home/test.txt') }

10
spec/lib/gitlab/ci/decompressed_gzip_size_validator_spec.rb
View File

@ -105,6 +105,16 @@ RSpec.describe Gitlab::Ci::DecompressedGzipSizeValidator, feature_category: :imp
end
end
context 'when archive path has multiple hard links' do
before do
FileUtils.link(filepath, File.join(Dir.mktmpdir, 'hard_link'))
end
it 'returns false' do
expect(subject).not_to be_valid
end
end
context 'when archive path is not a file' do
let(:filepath) { Dir.mktmpdir }
let(:filesize) { File.size(filepath) }

22
spec/lib/gitlab/github_import/attachments_downloader_spec.rb
View File

@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Gitlab::GithubImport::AttachmentsDownloader do
RSpec.describe Gitlab::GithubImport::AttachmentsDownloader, feature_category: :importers do
subject(:downloader) { described_class.new(file_url) }
let_it_be(:file_url) { 'https://example.com/avatar.png' }
@ -39,6 +39,26 @@ RSpec.describe Gitlab::GithubImport::AttachmentsDownloader do
end
end
context 'when file shares multiple hard links' do
let(:tmpdir) { Dir.mktmpdir }
let(:hard_link) { File.join(tmpdir, 'hard_link') }
before do
existing_file = File.join(tmpdir, 'file.txt')
FileUtils.touch(existing_file)
FileUtils.link(existing_file, hard_link)
allow(downloader).to receive(:filepath).and_return(hard_link)
end
it 'raises expected exception' do
expect(Gitlab::Utils::FileInfo).to receive(:linked?).with(hard_link).and_call_original
expect { downloader.perform }.to raise_exception(
described_class::DownloadError,
'Invalid downloaded file'
)
end
end
context 'when filename is malicious' do
let_it_be(:file_url) { 'https://example.com/ava%2F..%2Ftar.png' }

104
spec/lib/gitlab/import_export/command_line_util_spec.rb
View File

@ -5,13 +5,16 @@ require 'spec_helper'
RSpec.describe Gitlab::ImportExport::CommandLineUtil, feature_category: :importers do
include ExportFileHelper
let(:path) { "#{Dir.tmpdir}/symlink_test" }
let(:archive) { 'spec/fixtures/symlink_export.tar.gz' }
let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
let(:tmpdir) { Dir.mktmpdir }
# Separate where files are written during this test by their kind, to avoid them interfering with each other:
# - `source_dir` Dir to compress files from.
# - `target_dir` Dir to decompress archived files into.
# - `archive_dir` Dir to write any archive files to.
let(:source_dir) { Dir.mktmpdir }
let(:target_dir) { Dir.mktmpdir }
let(:archive_dir) { Dir.mktmpdir }
subject do
subject(:mock_class) do
Class.new do
include Gitlab::ImportExport::CommandLineUtil
@ -25,38 +28,59 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil, feature_category: :importe
end
before do
FileUtils.mkdir_p(path)
FileUtils.mkdir_p(source_dir)
end
after do
FileUtils.rm_rf(path)
FileUtils.rm_rf(source_dir)
FileUtils.rm_rf(target_dir)
FileUtils.rm_rf(archive_dir)
FileUtils.remove_entry(tmpdir)
end
shared_examples 'deletes symlinks' do |compression, decompression|
it 'deletes the symlinks', :aggregate_failures do
Dir.mkdir("#{tmpdir}/.git")
Dir.mkdir("#{tmpdir}/folder")
FileUtils.touch("#{tmpdir}/file.txt")
FileUtils.touch("#{tmpdir}/folder/file.txt")
FileUtils.touch("#{tmpdir}/.gitignore")
FileUtils.touch("#{tmpdir}/.git/config")
File.symlink('file.txt', "#{tmpdir}/.symlink")
File.symlink('file.txt', "#{tmpdir}/.git/.symlink")
File.symlink('file.txt', "#{tmpdir}/folder/.symlink")
archive = File.join(archive_dir, 'archive')
subject.public_send(compression, archive: archive, dir: tmpdir)
Dir.mkdir("#{source_dir}/.git")
Dir.mkdir("#{source_dir}/folder")
FileUtils.touch("#{source_dir}/file.txt")
FileUtils.touch("#{source_dir}/folder/file.txt")
FileUtils.touch("#{source_dir}/.gitignore")
FileUtils.touch("#{source_dir}/.git/config")
File.symlink('file.txt', "#{source_dir}/.symlink")
File.symlink('file.txt', "#{source_dir}/.git/.symlink")
File.symlink('file.txt', "#{source_dir}/folder/.symlink")
archive_file = File.join(archive_dir, 'symlink_archive.tar.gz')
subject.public_send(compression, archive: archive_file, dir: source_dir)
subject.public_send(decompression, archive: archive_file, dir: target_dir)
subject.public_send(decompression, archive: archive, dir: archive_dir)
expect(File).to exist("#{target_dir}/file.txt")
expect(File).to exist("#{target_dir}/folder/file.txt")
expect(File).to exist("#{target_dir}/.gitignore")
expect(File).to exist("#{target_dir}/.git/config")
expect(File).not_to exist("#{target_dir}/.symlink")
expect(File).not_to exist("#{target_dir}/.git/.symlink")
expect(File).not_to exist("#{target_dir}/folder/.symlink")
end
end
expect(File.exist?("#{archive_dir}/file.txt")).to eq(true)
expect(File.exist?("#{archive_dir}/folder/file.txt")).to eq(true)
expect(File.exist?("#{archive_dir}/.gitignore")).to eq(true)
expect(File.exist?("#{archive_dir}/.git/config")).to eq(true)
expect(File.exist?("#{archive_dir}/.symlink")).to eq(false)
expect(File.exist?("#{archive_dir}/.git/.symlink")).to eq(false)
expect(File.exist?("#{archive_dir}/folder/.symlink")).to eq(false)
shared_examples 'handles shared hard links' do |compression, decompression|
let(:archive_file) { File.join(archive_dir, 'hard_link_archive.tar.gz') }
subject(:decompress) { mock_class.public_send(decompression, archive: archive_file, dir: target_dir) }
before do
Dir.mkdir("#{source_dir}/dir")
FileUtils.touch("#{source_dir}/file.txt")
FileUtils.touch("#{source_dir}/dir/.file.txt")
FileUtils.link("#{source_dir}/file.txt", "#{source_dir}/.hard_linked_file.txt")
mock_class.public_send(compression, archive: archive_file, dir: source_dir)
end
it 'raises an exception and deletes the extraction dir', :aggregate_failures do
expect(FileUtils).to receive(:remove_dir).with(target_dir).and_call_original
expect(Dir).to exist(target_dir)
expect { decompress }.to raise_error(described_class::HardLinkError)
expect(Dir).not_to exist(target_dir)
end
end
@ -212,6 +236,8 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil, feature_category: :importe
end
describe '#gzip' do
let(:path) { source_dir }
it 'compresses specified file' do
tempfile = Tempfile.new('test', path)
filename = File.basename(tempfile.path)
@ -229,14 +255,16 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil, feature_category: :importe
end
describe '#gunzip' do
let(:path) { source_dir }
it 'decompresses specified file' do
filename = 'labels.ndjson.gz'
gz_filepath = "spec/fixtures/bulk_imports/gz/#{filename}"
FileUtils.copy_file(gz_filepath, File.join(tmpdir, filename))
FileUtils.copy_file(gz_filepath, File.join(path, filename))
subject.gunzip(dir: tmpdir, filename: filename)
subject.gunzip(dir: path, filename: filename)
expect(File.exist?(File.join(tmpdir, 'labels.ndjson'))).to eq(true)
expect(File.exist?(File.join(path, 'labels.ndjson'))).to eq(true)
end
context 'when exception occurs' do
@ -250,7 +278,7 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil, feature_category: :importe
it 'archives a folder without compression' do
archive_file = File.join(archive_dir, 'archive.tar')
result = subject.tar_cf(archive: archive_file, dir: tmpdir)
result = subject.tar_cf(archive: archive_file, dir: source_dir)
expect(result).to eq(true)
expect(File.exist?(archive_file)).to eq(true)
@ -270,29 +298,35 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil, feature_category: :importe
end
describe '#untar_zxf' do
let(:tar_archive_fixture) { 'spec/fixtures/symlink_export.tar.gz' }
it_behaves_like 'deletes symlinks', :tar_czf, :untar_zxf
it_behaves_like 'handles shared hard links', :tar_czf, :untar_zxf
it 'has the right mask for project.json' do
subject.untar_zxf(archive: archive, dir: path)
subject.untar_zxf(archive: tar_archive_fixture, dir: target_dir)
expect(file_permissions("#{path}/project.json")).to eq(0755) # originally 777
expect(file_permissions("#{target_dir}/project.json")).to eq(0755) # originally 777
end
it 'has the right mask for uploads' do
subject.untar_zxf(archive: archive, dir: path)
subject.untar_zxf(archive: tar_archive_fixture, dir: target_dir)
expect(file_permissions("#{path}/uploads")).to eq(0755) # originally 555
expect(file_permissions("#{target_dir}/uploads")).to eq(0755) # originally 555
end
end
describe '#untar_xf' do
let(:tar_archive_fixture) { 'spec/fixtures/symlink_export.tar.gz' }
it_behaves_like 'deletes symlinks', :tar_cf, :untar_xf
it_behaves_like 'handles shared hard links', :tar_cf, :untar_xf
it 'extracts archive without decompression' do
filename = 'archive.tar.gz'
archive_file = File.join(archive_dir, 'archive.tar')
FileUtils.copy_file(archive, File.join(archive_dir, filename))
FileUtils.copy_file(tar_archive_fixture, File.join(archive_dir, filename))
subject.gunzip(dir: archive_dir, filename: filename)
result = subject.untar_xf(archive: archive_file, dir: archive_dir)

17
spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb
View File

@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Gitlab::ImportExport::DecompressedArchiveSizeValidator do
RSpec.describe Gitlab::ImportExport::DecompressedArchiveSizeValidator, feature_category: :importers do
let_it_be(:filepath) { File.join(Dir.tmpdir, 'decompressed_archive_size_validator_spec.gz') }
before(:all) do
@ -121,7 +121,7 @@ RSpec.describe Gitlab::ImportExport::DecompressedArchiveSizeValidator do
context 'which archive path is a symlink' do
let(:filepath) { File.join(Dir.tmpdir, 'symlink') }
let(:error_message) { 'Archive path is a symlink' }
let(:error_message) { 'Archive path is a symlink or hard link' }
before do
FileUtils.ln_s(filepath, filepath, force: true)
@ -132,6 +132,19 @@ RSpec.describe Gitlab::ImportExport::DecompressedArchiveSizeValidator do
end
end
context 'when archive path shares multiple hard links' do
let(:filesize) { 32 }
let(:error_message) { 'Archive path is a symlink or hard link' }
before do
FileUtils.link(filepath, File.join(Dir.mktmpdir, 'hard_link'))
end
it 'returns false' do
expect(subject).not_to be_valid
end
end
context 'when archive path is not a file' do
let(:filepath) { Dir.mktmpdir }
let(:filesize) { File.size(filepath) }

57
spec/lib/gitlab/import_export/file_importer_spec.rb
View File

@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Gitlab::ImportExport::FileImporter do
RSpec.describe Gitlab::ImportExport::FileImporter, feature_category: :importers do
include ExportFileHelper
let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
@ -113,28 +113,73 @@ RSpec.describe Gitlab::ImportExport::FileImporter do
end
context 'error' do
subject(:import) { described_class.import(importable: build(:project), archive_file: '', shared: shared) }
before do
allow_next_instance_of(described_class) do |instance|
allow(instance).to receive(:wait_for_archived_file).and_raise(StandardError)
allow(instance).to receive(:wait_for_archived_file).and_raise(StandardError, 'foo')
end
described_class.import(importable: build(:project), archive_file: '', shared: shared)
end
it 'removes symlinks in root folder' do
import
expect(File.exist?(symlink_file)).to be false
end
it 'removes hidden symlinks in root folder' do
import
expect(File.exist?(hidden_symlink_file)).to be false
end
it 'removes symlinks in subfolders' do
import
expect(File.exist?(subfolder_symlink_file)).to be false
end
it 'does not remove a valid file' do
import
expect(File.exist?(valid_file)).to be true
end
it 'returns false and sets an error on shared' do
result = import
expect(result).to eq(false)
expect(shared.errors.join).to eq('foo')
end
context 'when files in the archive share hard links' do
let(:hard_link_file) { "#{shared.export_path}/hard_link_file.txt" }
before do
FileUtils.link(valid_file, hard_link_file)
end
it 'returns false and sets an error on shared' do
result = import
expect(result).to eq(false)
expect(shared.errors.join).to eq('File shares hard link')
end
it 'removes all files in export path' do
expect(Dir).to exist(shared.export_path)
expect(File).to exist(symlink_file)
expect(File).to exist(hard_link_file)
expect(File).to exist(valid_file)
import
expect(File).not_to exist(symlink_file)
expect(File).not_to exist(hard_link_file)
expect(File).not_to exist(valid_file)
expect(Dir).not_to exist(shared.export_path)
end
end
end
context 'when file exceeds acceptable decompressed size' do
@ -157,8 +202,10 @@ RSpec.describe Gitlab::ImportExport::FileImporter do
allow(Gitlab::ImportExport::DecompressedArchiveSizeValidator).to receive(:max_bytes).and_return(1)
end
it 'returns false' do
expect(subject.import).to eq(false)
it 'returns false and sets an error on shared' do
result = subject.import
expect(result).to eq(false)
expect(shared.errors.join).to eq('Decompressed archive size validation failed.')
end
end

46
spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb
View File

@ -35,16 +35,22 @@ RSpec.describe Gitlab::ImportExport::Json::NdjsonReader, feature_category: :impo
expect(subject).to eq(root_tree)
end
context 'when project.json is symlink' do
it 'raises error an error' do
Dir.mktmpdir do |tmpdir|
FileUtils.touch(File.join(tmpdir, 'passwd'))
File.symlink(File.join(tmpdir, 'passwd'), File.join(tmpdir, 'project.json'))
context 'when project.json is symlink or hard link' do
using RSpec::Parameterized::TableSyntax
ndjson_reader = described_class.new(tmpdir)
where(:link_method) { [:link, :symlink] }
expect { ndjson_reader.consume_attributes(importable_path) }
.to raise_error(Gitlab::ImportExport::Error, 'Invalid file')
with_them do
it 'raises an error' do
Dir.mktmpdir do |tmpdir|
FileUtils.touch(File.join(tmpdir, 'passwd'))
FileUtils.send(link_method, File.join(tmpdir, 'passwd'), File.join(tmpdir, 'project.json'))
ndjson_reader = described_class.new(tmpdir)
expect { ndjson_reader.consume_attributes(importable_path) }
.to raise_error(Gitlab::ImportExport::Error, 'Invalid file')
end
end
end
end
@ -97,18 +103,24 @@ RSpec.describe Gitlab::ImportExport::Json::NdjsonReader, feature_category: :impo
end
end
context 'when relation file is a symlink' do
it 'yields nothing to the Enumerator' do
Dir.mktmpdir do |tmpdir|
Dir.mkdir(File.join(tmpdir, 'project'))
File.write(File.join(tmpdir, 'passwd'), "{}\n{}")
File.symlink(File.join(tmpdir, 'passwd'), File.join(tmpdir, 'project', 'issues.ndjson'))
context 'when relation file is a symlink or hard link' do
using RSpec::Parameterized::TableSyntax
ndjson_reader = described_class.new(tmpdir)
where(:link_method) { [:link, :symlink] }
result = ndjson_reader.consume_relation(importable_path, 'issues')
with_them do
it 'yields nothing to the Enumerator' do
Dir.mktmpdir do |tmpdir|
Dir.mkdir(File.join(tmpdir, 'project'))
File.write(File.join(tmpdir, 'passwd'), "{}\n{}")
FileUtils.send(link_method, File.join(tmpdir, 'passwd'), File.join(tmpdir, 'project', 'issues.ndjson'))
expect(result.to_a).to eq([])
ndjson_reader = described_class.new(tmpdir)
result = ndjson_reader.consume_relation(importable_path, 'issues')
expect(result.to_a).to eq([])
end
end
end
end

4
spec/lib/gitlab/import_export/recursive_merge_folders_spec.rb
View File

@ -4,15 +4,17 @@ require 'spec_helper'
RSpec.describe Gitlab::ImportExport::RecursiveMergeFolders do
describe '.merge' do
it 'merge folder and ignore symlinks' do
it 'merges folder and ignores symlinks and files that share hard links' do
Dir.mktmpdir do |tmpdir|
source = "#{tmpdir}/source"
FileUtils.mkdir_p("#{source}/folder/folder")
FileUtils.touch("#{source}/file1.txt")
FileUtils.touch("#{source}/file_that_shares_hard_links.txt")
FileUtils.touch("#{source}/folder/file2.txt")
FileUtils.touch("#{source}/folder/folder/file3.txt")
FileUtils.ln_s("#{source}/file1.txt", "#{source}/symlink-file1.txt")
FileUtils.ln_s("#{source}/folder", "#{source}/symlink-folder")
FileUtils.link("#{source}/file_that_shares_hard_links.txt", "#{source}/hard_link.txt")
target = "#{tmpdir}/target"
FileUtils.mkdir_p("#{target}/folder/folder")

59
spec/lib/gitlab/plantuml_spec.rb Normal file
View File

@ -0,0 +1,59 @@
# frozen_string_literal: true
require "spec_helper"
RSpec.describe Gitlab::Plantuml, feature_category: :shared do
describe ".configure" do
subject { described_class.configure }
let(:plantuml_url) { "http://plantuml.foo.bar" }
before do
allow(Gitlab::CurrentSettings).to receive(:plantuml_url).and_return(plantuml_url)
end
context "when PlantUML is enabled" do
before do
allow(Gitlab::CurrentSettings).to receive(:plantuml_enabled).and_return(true)
end
it "configures the endpoint URL" do
expect(subject.url).to eq(plantuml_url)
end
it "enables PNG support" do
expect(subject.png_enable).to be_truthy
end
it "disables SVG support" do
expect(subject.svg_enable).to be_falsey
end
it "disables TXT support" do
expect(subject.txt_enable).to be_falsey
end
end
context "when PlantUML is disabled" do
before do
allow(Gitlab::CurrentSettings).to receive(:plantuml_enabled).and_return(false)
end
it "configures the endpoint URL" do
expect(subject.url).to eq(plantuml_url)
end
it "enables PNG support" do
expect(subject.png_enable).to be_falsey
end
it "disables SVG support" do
expect(subject.svg_enable).to be_falsey
end
it "disables TXT support" do
expect(subject.txt_enable).to be_falsey
end
end
end
end

88
spec/lib/gitlab/utils/file_info_spec.rb Normal file
View File

@ -0,0 +1,88 @@
# frozen_string_literal: true
require 'fast_spec_helper'
RSpec.describe Gitlab::Utils::FileInfo, feature_category: :shared do
let(:tmpdir) { Dir.mktmpdir }
let(:file_path) { "#{tmpdir}/test.txt" }
before do
FileUtils.touch(file_path)
end
after do
FileUtils.rm_rf(tmpdir)
end
describe '.linked?' do
it 'raises an error when file does not exist' do
expect { subject.linked?('foo') }.to raise_error(Errno::ENOENT)
end
shared_examples 'identifies a linked file' do
it 'returns false when file or dir is not a link' do
expect(subject.linked?(tmpdir)).to eq(false)
expect(subject.linked?(file)).to eq(false)
end
it 'returns true when file or dir is symlinked' do
FileUtils.symlink(tmpdir, "#{tmpdir}/symlinked_dir")
FileUtils.symlink(file_path, "#{tmpdir}/symlinked_file.txt")
expect(subject.linked?("#{tmpdir}/symlinked_dir")).to eq(true)
expect(subject.linked?("#{tmpdir}/symlinked_file.txt")).to eq(true)
end
it 'returns true when file has more than one hard link' do
FileUtils.link(file_path, "#{tmpdir}/hardlinked_file.txt")
expect(subject.linked?(file)).to eq(true)
expect(subject.linked?("#{tmpdir}/hardlinked_file.txt")).to eq(true)
end
end
context 'when file is a File::Stat' do
let(:file) { File.lstat(file_path) }
it_behaves_like 'identifies a linked file'
end
context 'when file is path' do
let(:file) { file_path }
it_behaves_like 'identifies a linked file'
end
end
describe '.shares_hard_link?' do
it 'raises an error when file does not exist' do
expect { subject.shares_hard_link?('foo') }.to raise_error(Errno::ENOENT)
end
shared_examples 'identifies a file that shares a hard link' do
it 'returns false when file or dir does not share hard links' do
expect(subject.shares_hard_link?(tmpdir)).to eq(false)
expect(subject.shares_hard_link?(file)).to eq(false)
end
it 'returns true when file has more than one hard link' do
FileUtils.link(file_path, "#{tmpdir}/hardlinked_file.txt")
expect(subject.shares_hard_link?(file)).to eq(true)
expect(subject.shares_hard_link?("#{tmpdir}/hardlinked_file.txt")).to eq(true)
end
end
context 'when file is a File::Stat' do
let(:file) { File.lstat(file_path) }
it_behaves_like 'identifies a file that shares a hard link'
end
context 'when file is path' do
let(:file) { file_path }
it_behaves_like 'identifies a file that shares a hard link'
end
end
end

10
spec/services/bulk_imports/archive_extraction_service_spec.rb
View File

@ -43,13 +43,21 @@ RSpec.describe BulkImports::ArchiveExtractionService, feature_category: :importe
context 'when archive file is a symlink' do
it 'raises an error' do
FileUtils.ln_s(File.join(tmpdir, filename), File.join(tmpdir, 'symlink'))
FileUtils.ln_s(filepath, File.join(tmpdir, 'symlink'))
expect { described_class.new(tmpdir: tmpdir, filename: 'symlink').execute }
.to raise_error(BulkImports::Error, 'Invalid file')
end
end
context 'when archive file shares multiple hard links' do
it 'raises an error' do
FileUtils.link(filepath, File.join(tmpdir, 'hard_link'))
expect { subject.execute }.to raise_error(BulkImports::Error, 'Invalid file')
end
end
context 'when filepath is being traversed' do
it 'raises an error' do
expect { described_class.new(tmpdir: File.join(Dir.mktmpdir, 'test', '..'), filename: 'name').execute }

73
spec/services/bulk_imports/file_decompression_service_spec.rb
View File

@ -3,6 +3,8 @@
require 'spec_helper'
RSpec.describe BulkImports::FileDecompressionService, feature_category: :importers do
using RSpec::Parameterized::TableSyntax
let_it_be(:tmpdir) { Dir.mktmpdir }
let_it_be(:ndjson_filename) { 'labels.ndjson' }
let_it_be(:ndjson_filepath) { File.join(tmpdir, ndjson_filename) }
@ -70,39 +72,68 @@ RSpec.describe BulkImports::FileDecompressionService, feature_category: :importe
end
end
context 'when compressed file is a symlink' do
let_it_be(:symlink) { File.join(tmpdir, 'symlink.gz') }
before do
FileUtils.ln_s(File.join(tmpdir, gz_filename), symlink)
end
subject { described_class.new(tmpdir: tmpdir, filename: 'symlink.gz') }
it 'raises an error and removes the file' do
shared_examples 'raises an error and removes the file' do |error_message:|
specify do
expect { subject.execute }
.to raise_error(BulkImports::FileDecompressionService::ServiceError, 'File decompression error')
expect(File.exist?(symlink)).to eq(false)
.to raise_error(BulkImports::FileDecompressionService::ServiceError, error_message)
expect(File).not_to exist(file)
end
end
context 'when decompressed file is a symlink' do
let_it_be(:symlink) { File.join(tmpdir, 'symlink') }
shared_context 'when compressed file' do
let_it_be(:file) { File.join(tmpdir, 'file.gz') }
subject { described_class.new(tmpdir: tmpdir, filename: 'file.gz') }
before do
FileUtils.ln_s(File.join(tmpdir, ndjson_filename), symlink)
subject.instance_variable_set(:@decompressed_filepath, symlink)
FileUtils.send(link_method, File.join(tmpdir, gz_filename), file)
end
end
shared_context 'when decompressed file' do
let_it_be(:file) { File.join(tmpdir, 'file.txt') }
subject { described_class.new(tmpdir: tmpdir, filename: gz_filename) }
it 'raises an error and removes the file' do
expect { subject.execute }.to raise_error(described_class::ServiceError, 'Invalid file')
before do
original_file = File.join(tmpdir, 'original_file.txt')
FileUtils.touch(original_file)
FileUtils.send(link_method, original_file, file)
expect(File.exist?(symlink)).to eq(false)
subject.instance_variable_set(:@decompressed_filepath, file)
end
end
context 'when compressed file is a symlink' do
let(:link_method) { :symlink }
include_context 'when compressed file'
include_examples 'raises an error and removes the file', error_message: 'File decompression error'
end
context 'when compressed file shares multiple hard links' do
let(:link_method) { :link }
include_context 'when compressed file'
include_examples 'raises an error and removes the file', error_message: 'File decompression error'
end
context 'when decompressed file is a symlink' do
let(:link_method) { :symlink }
include_context 'when decompressed file'
include_examples 'raises an error and removes the file', error_message: 'Invalid file'
end
context 'when decompressed file shares multiple hard links' do
let(:link_method) { :link }
include_context 'when decompressed file'
include_examples 'raises an error and removes the file', error_message: 'Invalid file'
end
end
end

32
spec/services/bulk_imports/file_download_service_spec.rb
View File

@ -10,7 +10,7 @@ RSpec.describe BulkImports::FileDownloadService, feature_category: :importers do
let_it_be(:content_type) { 'application/octet-stream' }
let_it_be(:content_disposition) { nil }
let_it_be(:filename) { 'file_download_service_spec' }
let_it_be(:tmpdir) { Dir.tmpdir }
let_it_be(:tmpdir) { Dir.mktmpdir }
let_it_be(:filepath) { File.join(tmpdir, filename) }
let_it_be(:content_length) { 1000 }
@ -247,6 +247,36 @@ RSpec.describe BulkImports::FileDownloadService, feature_category: :importers do
end
end
context 'when file shares multiple hard links' do
let_it_be(:hard_link) { File.join(tmpdir, 'hard_link') }
before do
existing_file = File.join(Dir.mktmpdir, filename)
FileUtils.touch(existing_file)
FileUtils.link(existing_file, hard_link)
end
subject do
described_class.new(
configuration: config,
relative_url: '/test',
tmpdir: tmpdir,
filename: 'hard_link',
file_size_limit: file_size_limit,
allowed_content_types: allowed_content_types
)
end
it 'raises an error and removes the file' do
expect { subject.execute }.to raise_error(
described_class::ServiceError,
'Invalid downloaded file'
)
expect(File.exist?(hard_link)).to eq(false)
end
end
context 'when dir is not in tmpdir' do
subject do
described_class.new(