2019-10-12 23:23:16 +08:00
|
|
|
# frozen_string_literal: true
|
2022-01-13 17:05:07 +08:00
|
|
|
|
2023-12-14 17:47:17 +08:00
|
|
|
require "test_helper"
|
2017-01-19 00:28:16 +08:00
|
|
|
|
|
|
|
class CompileCacheTest < Minitest::Test
|
2023-12-14 17:47:17 +08:00
|
|
|
include CompileCacheISeqHelper
|
|
|
|
include TmpdirHelper
|
2017-01-19 00:28:16 +08:00
|
|
|
|
2022-11-24 23:20:57 +08:00
|
|
|
def teardown
|
|
|
|
super
|
|
|
|
Bootsnap::CompileCache::Native.readonly = false
|
2024-01-31 15:51:15 +08:00
|
|
|
Bootsnap::CompileCache::Native.revalidation = false
|
Revalidate stale entries using a digest.
Ref: https://github.com/Shopify/bootsnap/issues/336
Bootsnap was initially designed for improving boot time
in development, so it was logical to use `mtime` to detect changes
given that's reliable on a given machine.
But is just as useful on production and CI environments, however
there its hit rate can vary a lot because depending on how the
source code and caches are saved and restored, many if not all
`mtime` will have changed.
To improve this, we can first try to revalidate using the `mtime`,
and if it fails, fallback to compare a digest of the file content.
Digesting a file, even with `fnv1a_64` is of course an overhead,
but the assumption is that true misses should be relatively rare
and that digesting the file will always be faster than compiling it.
So even if it only improve the hit rate marginally, it should be
faster overall.
Also we only recompute the digest if the file mtime changed, but
its size remained the same, which should discard the overwhelming
majority of legitimate source file changes.
Co-authored-by: Jean Boussier <jean.boussier@gmail.com>
2024-01-29 23:24:20 +08:00
|
|
|
Bootsnap.instrumentation = nil
|
2022-11-24 23:20:57 +08:00
|
|
|
end
|
|
|
|
|
2017-06-22 23:45:00 +08:00
|
|
|
def test_compile_option_crc32
|
|
|
|
# Just assert that this works.
|
|
|
|
Bootsnap::CompileCache::Native.compile_option_crc32 = 0xffffffff
|
|
|
|
assert_raises(RangeError) do
|
|
|
|
Bootsnap::CompileCache::Native.compile_option_crc32 = 0xffffffff + 1
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2024-03-05 21:07:32 +08:00
|
|
|
def test_coverage_running
|
2023-12-14 17:47:17 +08:00
|
|
|
require "coverage"
|
2024-03-05 21:07:32 +08:00
|
|
|
Bootsnap::CompileCache::ISeq.expects(:fetch).times(0)
|
2017-06-20 03:17:29 +08:00
|
|
|
begin
|
|
|
|
Coverage.start
|
2024-03-05 21:07:32 +08:00
|
|
|
path = Help.set_file("a.rb", "a = a = 3", 100)
|
|
|
|
load(path)
|
2017-06-20 03:17:29 +08:00
|
|
|
ensure
|
|
|
|
Coverage.result
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-05-24 12:06:48 +08:00
|
|
|
def test_no_write_permission_to_cache
|
2022-01-13 17:05:07 +08:00
|
|
|
if RbConfig::CONFIG["host_os"] =~ /mswin|mingw|cygwin/
|
2020-09-05 10:23:09 +08:00
|
|
|
# Always pass this test on Windows because directories aren't read, only
|
|
|
|
# listed. You can restrict the ability to list directory contents on
|
|
|
|
# Windows or you can set ACLS on a folder such that it is not allowed to
|
|
|
|
# list contents.
|
|
|
|
#
|
|
|
|
# Since we can't read directories on windows, this specific test doesn't
|
2023-07-22 07:00:12 +08:00
|
|
|
# make sense. In addition we test read-only files in
|
2020-09-05 10:23:09 +08:00
|
|
|
# `test_can_open_read_only_cache` so we are covered testing reading
|
|
|
|
# read-only files.
|
|
|
|
pass
|
|
|
|
else
|
2022-01-15 01:21:19 +08:00
|
|
|
path = Help.set_file("a.rb", "a = a = 3", 100)
|
2020-09-05 10:23:09 +08:00
|
|
|
folder = File.dirname(Help.cache_path(@tmp_dir, path))
|
|
|
|
FileUtils.mkdir_p(folder)
|
2022-01-13 17:05:07 +08:00
|
|
|
FileUtils.chmod(0o400, folder)
|
2021-04-16 16:33:31 +08:00
|
|
|
load(path)
|
2020-09-05 10:23:09 +08:00
|
|
|
end
|
2017-01-19 00:28:16 +08:00
|
|
|
end
|
|
|
|
|
2023-07-22 07:00:12 +08:00
|
|
|
def test_no_read_permission
|
|
|
|
if RbConfig::CONFIG["host_os"] =~ /mswin|mingw|cygwin/
|
|
|
|
# On windows removing read permission doesn't prevent reading.
|
|
|
|
pass
|
|
|
|
else
|
|
|
|
path = Help.set_file("a.rb", "a = a = 3", 100)
|
|
|
|
FileUtils.chmod(0o000, path)
|
|
|
|
exception = assert_raises(LoadError) do
|
|
|
|
load(path)
|
|
|
|
end
|
|
|
|
assert_match(path, exception.message)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-06-04 10:20:11 +08:00
|
|
|
def test_can_open_read_only_cache
|
2022-01-13 17:05:07 +08:00
|
|
|
path = Help.set_file("a.rb", "a = a = 3", 100)
|
2017-06-04 10:20:11 +08:00
|
|
|
# Load once to create the cache file
|
|
|
|
load(path)
|
2022-01-13 17:05:07 +08:00
|
|
|
FileUtils.chmod(0o400, path)
|
2017-06-04 10:20:11 +08:00
|
|
|
# Loading again after the file is marked read-only should still succeed
|
|
|
|
load(path)
|
|
|
|
end
|
|
|
|
|
2017-01-19 00:28:16 +08:00
|
|
|
def test_file_is_only_read_once
|
2022-01-13 17:05:07 +08:00
|
|
|
path = Help.set_file("a.rb", "a = a = 3", 100)
|
2017-01-19 00:28:16 +08:00
|
|
|
storage = RubyVM::InstructionSequence.compile_file(path).to_binary
|
|
|
|
output = RubyVM::InstructionSequence.load_from_binary(storage)
|
|
|
|
# This doesn't really *prove* the file is only read once, but
|
|
|
|
# it at least asserts the input is only cached once.
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:input_to_storage).times(1).returns(storage)
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:storage_to_output).times(2).returns(output)
|
|
|
|
load(path)
|
|
|
|
load(path)
|
|
|
|
end
|
|
|
|
|
|
|
|
def test_raises_syntax_error
|
2022-01-13 17:05:07 +08:00
|
|
|
path = Help.set_file("a.rb", "a = (3", 100)
|
2017-01-19 00:28:16 +08:00
|
|
|
assert_raises(SyntaxError) do
|
|
|
|
# SyntaxError emits directly to stderr in addition to raising, it seems.
|
2017-05-24 12:06:48 +08:00
|
|
|
capture_io { load(path) }
|
2017-01-19 00:28:16 +08:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-05-24 12:06:48 +08:00
|
|
|
def test_no_recache_when_mtime_and_size_same
|
2022-01-13 17:05:07 +08:00
|
|
|
path = Help.set_file("a.rb", "a = a = 3", 100)
|
2017-01-19 00:28:16 +08:00
|
|
|
storage = RubyVM::InstructionSequence.compile_file(path).to_binary
|
|
|
|
output = RubyVM::InstructionSequence.load_from_binary(storage)
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:input_to_storage).times(1).returns(storage)
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:storage_to_output).times(2).returns(output)
|
|
|
|
|
|
|
|
load(path)
|
2022-01-13 17:05:07 +08:00
|
|
|
Help.set_file(path, "a = a = 4", 100)
|
2017-01-19 00:28:16 +08:00
|
|
|
load(path)
|
|
|
|
end
|
|
|
|
|
2017-05-24 12:06:48 +08:00
|
|
|
def test_recache_when_mtime_different
|
2022-01-13 17:05:07 +08:00
|
|
|
path = Help.set_file("a.rb", "a = a = 3", 100)
|
2017-01-19 00:28:16 +08:00
|
|
|
storage = RubyVM::InstructionSequence.compile_file(path).to_binary
|
|
|
|
output = RubyVM::InstructionSequence.load_from_binary(storage)
|
|
|
|
# Totally lies the second time but that's not the point.
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:input_to_storage).times(2).returns(storage)
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:storage_to_output).times(2).returns(output)
|
|
|
|
|
|
|
|
load(path)
|
2022-01-13 17:05:07 +08:00
|
|
|
Help.set_file(path, "a = a = 2", 101)
|
2017-01-19 00:28:16 +08:00
|
|
|
load(path)
|
|
|
|
end
|
|
|
|
|
2017-05-24 12:06:48 +08:00
|
|
|
def test_recache_when_size_different
|
2022-01-13 17:05:07 +08:00
|
|
|
path = Help.set_file("a.rb", "a = a = 3", 100)
|
2017-01-19 00:28:16 +08:00
|
|
|
storage = RubyVM::InstructionSequence.compile_file(path).to_binary
|
|
|
|
output = RubyVM::InstructionSequence.load_from_binary(storage)
|
2017-05-24 12:06:48 +08:00
|
|
|
# Totally lies the second time but that's not the point.
|
2017-01-19 00:28:16 +08:00
|
|
|
Bootsnap::CompileCache::ISeq.expects(:input_to_storage).times(2).returns(storage)
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:storage_to_output).times(2).returns(output)
|
2017-05-24 12:06:48 +08:00
|
|
|
|
2017-01-19 00:28:16 +08:00
|
|
|
load(path)
|
2022-01-13 17:05:07 +08:00
|
|
|
Help.set_file(path, "a = 33", 100)
|
2017-01-19 00:28:16 +08:00
|
|
|
load(path)
|
|
|
|
end
|
|
|
|
|
2022-11-24 23:20:57 +08:00
|
|
|
def test_dont_store_cache_after_a_miss_when_readonly
|
|
|
|
Bootsnap::CompileCache::Native.readonly = true
|
|
|
|
|
|
|
|
path = Help.set_file("a.rb", "a = a = 3", 100)
|
|
|
|
output = RubyVM::InstructionSequence.compile_file(path)
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:input_to_storage).never
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:storage_to_output).never
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:input_to_output).once.returns(output)
|
|
|
|
|
|
|
|
load(path)
|
|
|
|
end
|
|
|
|
|
|
|
|
def test_dont_store_cache_after_a_stale_when_readonly
|
|
|
|
path = Help.set_file("a.rb", "a = a = 3", 100)
|
|
|
|
load(path)
|
|
|
|
|
|
|
|
Bootsnap::CompileCache::Native.readonly = true
|
|
|
|
|
|
|
|
output = RubyVM::InstructionSequence.compile_file(path)
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:input_to_storage).never
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:storage_to_output).once.returns(output)
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:input_to_output).never
|
|
|
|
|
|
|
|
load(path)
|
|
|
|
end
|
|
|
|
|
2024-01-31 16:38:11 +08:00
|
|
|
def test_revalidation
|
|
|
|
Bootsnap::CompileCache::Native.revalidation = true
|
|
|
|
|
|
|
|
file_path = Help.set_file("a.rb", "a = a = 3", 100)
|
|
|
|
load(file_path)
|
|
|
|
|
|
|
|
calls = []
|
|
|
|
Bootsnap.instrumentation = ->(event, path) { calls << [event, path] }
|
|
|
|
|
|
|
|
5.times do
|
|
|
|
FileUtils.touch("a.rb", mtime: File.mtime("a.rb") + 42)
|
|
|
|
load(file_path)
|
|
|
|
load(file_path)
|
|
|
|
end
|
|
|
|
|
|
|
|
assert_equal [[:revalidated, "a.rb"], [:hit, "a.rb"]] * 5, calls
|
|
|
|
end
|
|
|
|
|
Revalidate stale entries using a digest.
Ref: https://github.com/Shopify/bootsnap/issues/336
Bootsnap was initially designed for improving boot time
in development, so it was logical to use `mtime` to detect changes
given that's reliable on a given machine.
But is just as useful on production and CI environments, however
there its hit rate can vary a lot because depending on how the
source code and caches are saved and restored, many if not all
`mtime` will have changed.
To improve this, we can first try to revalidate using the `mtime`,
and if it fails, fallback to compare a digest of the file content.
Digesting a file, even with `fnv1a_64` is of course an overhead,
but the assumption is that true misses should be relatively rare
and that digesting the file will always be faster than compiling it.
So even if it only improve the hit rate marginally, it should be
faster overall.
Also we only recompute the digest if the file mtime changed, but
its size remained the same, which should discard the overwhelming
majority of legitimate source file changes.
Co-authored-by: Jean Boussier <jean.boussier@gmail.com>
2024-01-29 23:24:20 +08:00
|
|
|
def test_dont_revalidate_when_readonly
|
2024-01-31 15:51:15 +08:00
|
|
|
Bootsnap::CompileCache::Native.revalidation = true
|
|
|
|
|
Revalidate stale entries using a digest.
Ref: https://github.com/Shopify/bootsnap/issues/336
Bootsnap was initially designed for improving boot time
in development, so it was logical to use `mtime` to detect changes
given that's reliable on a given machine.
But is just as useful on production and CI environments, however
there its hit rate can vary a lot because depending on how the
source code and caches are saved and restored, many if not all
`mtime` will have changed.
To improve this, we can first try to revalidate using the `mtime`,
and if it fails, fallback to compare a digest of the file content.
Digesting a file, even with `fnv1a_64` is of course an overhead,
but the assumption is that true misses should be relatively rare
and that digesting the file will always be faster than compiling it.
So even if it only improve the hit rate marginally, it should be
faster overall.
Also we only recompute the digest if the file mtime changed, but
its size remained the same, which should discard the overwhelming
majority of legitimate source file changes.
Co-authored-by: Jean Boussier <jean.boussier@gmail.com>
2024-01-29 23:24:20 +08:00
|
|
|
path = Help.set_file("a.rb", "a = a = 3", 100)
|
|
|
|
load(path)
|
|
|
|
|
|
|
|
entries = Dir["#{Bootsnap::CompileCache::ISeq.cache_dir}/**/*"].select { |f| File.file?(f) }
|
|
|
|
assert_equal 1, entries.size
|
|
|
|
cache_entry = entries.first
|
|
|
|
old_cache_content = File.binread(cache_entry)
|
|
|
|
|
|
|
|
Bootsnap::CompileCache::Native.readonly = true
|
|
|
|
|
|
|
|
output = RubyVM::InstructionSequence.compile_file(path)
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:input_to_storage).never
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:storage_to_output).once.returns(output)
|
|
|
|
Bootsnap::CompileCache::ISeq.expects(:input_to_output).never
|
|
|
|
|
|
|
|
FileUtils.touch(path, mtime: File.mtime(path) + 50)
|
|
|
|
|
|
|
|
calls = []
|
|
|
|
Bootsnap.instrumentation = ->(event, source_path) { calls << [event, source_path] }
|
|
|
|
load(path)
|
|
|
|
|
|
|
|
assert_equal [[:revalidated, "a.rb"]], calls
|
|
|
|
|
|
|
|
new_cache_content = File.binread(cache_entry)
|
|
|
|
assert_equal old_cache_content, new_cache_content, "Cache entry was mutated"
|
|
|
|
end
|
|
|
|
|
2017-05-24 12:06:48 +08:00
|
|
|
def test_invalid_cache_file
|
2022-01-13 17:05:07 +08:00
|
|
|
path = Help.set_file("a.rb", "a = a = 3", 100)
|
2022-01-13 19:51:39 +08:00
|
|
|
cp = Help.cache_path("#{@tmp_dir}-iseq", path)
|
2017-05-24 12:06:48 +08:00
|
|
|
FileUtils.mkdir_p(File.dirname(cp))
|
2022-01-13 17:05:07 +08:00
|
|
|
File.write(cp, "nope")
|
2017-05-24 12:06:48 +08:00
|
|
|
load(path)
|
|
|
|
assert(File.size(cp) > 32) # cache was overwritten
|
2017-01-19 00:28:16 +08:00
|
|
|
end
|
2021-02-01 18:41:46 +08:00
|
|
|
|
|
|
|
def test_instrumentation_hit
|
2022-01-13 17:05:07 +08:00
|
|
|
file_path = Help.set_file("a.rb", "a = a = 3", 100)
|
|
|
|
load(file_path)
|
2021-02-01 18:41:46 +08:00
|
|
|
|
|
|
|
calls = []
|
|
|
|
Bootsnap.instrumentation = ->(event, path) { calls << [event, path] }
|
|
|
|
|
2022-01-13 17:05:07 +08:00
|
|
|
load(file_path)
|
2021-02-01 18:41:46 +08:00
|
|
|
|
2024-01-30 21:04:13 +08:00
|
|
|
assert_equal [[:hit, "a.rb"]], calls
|
2021-02-01 18:41:46 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
def test_instrumentation_miss
|
2022-01-13 17:05:07 +08:00
|
|
|
file_path = Help.set_file("a.rb", "a = a = 3", 100)
|
2021-02-01 18:41:46 +08:00
|
|
|
|
|
|
|
calls = []
|
|
|
|
Bootsnap.instrumentation = ->(event, path) { calls << [event, path] }
|
|
|
|
|
2022-01-13 17:05:07 +08:00
|
|
|
load(file_path)
|
2021-02-01 18:41:46 +08:00
|
|
|
|
2022-01-13 17:05:07 +08:00
|
|
|
assert_equal [[:miss, "a.rb"]], calls
|
Revalidate stale entries using a digest.
Ref: https://github.com/Shopify/bootsnap/issues/336
Bootsnap was initially designed for improving boot time
in development, so it was logical to use `mtime` to detect changes
given that's reliable on a given machine.
But is just as useful on production and CI environments, however
there its hit rate can vary a lot because depending on how the
source code and caches are saved and restored, many if not all
`mtime` will have changed.
To improve this, we can first try to revalidate using the `mtime`,
and if it fails, fallback to compare a digest of the file content.
Digesting a file, even with `fnv1a_64` is of course an overhead,
but the assumption is that true misses should be relatively rare
and that digesting the file will always be faster than compiling it.
So even if it only improve the hit rate marginally, it should be
faster overall.
Also we only recompute the digest if the file mtime changed, but
its size remained the same, which should discard the overwhelming
majority of legitimate source file changes.
Co-authored-by: Jean Boussier <jean.boussier@gmail.com>
2024-01-29 23:24:20 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
def test_instrumentation_revalidate
|
2024-01-31 15:51:15 +08:00
|
|
|
Bootsnap::CompileCache::Native.revalidation = true
|
|
|
|
|
Revalidate stale entries using a digest.
Ref: https://github.com/Shopify/bootsnap/issues/336
Bootsnap was initially designed for improving boot time
in development, so it was logical to use `mtime` to detect changes
given that's reliable on a given machine.
But is just as useful on production and CI environments, however
there its hit rate can vary a lot because depending on how the
source code and caches are saved and restored, many if not all
`mtime` will have changed.
To improve this, we can first try to revalidate using the `mtime`,
and if it fails, fallback to compare a digest of the file content.
Digesting a file, even with `fnv1a_64` is of course an overhead,
but the assumption is that true misses should be relatively rare
and that digesting the file will always be faster than compiling it.
So even if it only improve the hit rate marginally, it should be
faster overall.
Also we only recompute the digest if the file mtime changed, but
its size remained the same, which should discard the overwhelming
majority of legitimate source file changes.
Co-authored-by: Jean Boussier <jean.boussier@gmail.com>
2024-01-29 23:24:20 +08:00
|
|
|
file_path = Help.set_file("a.rb", "a = a = 3", 100)
|
|
|
|
load(file_path)
|
|
|
|
FileUtils.touch("a.rb", mtime: File.mtime("a.rb") + 42)
|
|
|
|
|
|
|
|
calls = []
|
|
|
|
Bootsnap.instrumentation = ->(event, path) { calls << [event, path] }
|
|
|
|
|
|
|
|
load(file_path)
|
|
|
|
|
|
|
|
assert_equal [[:revalidated, "a.rb"]], calls
|
2021-02-01 18:41:46 +08:00
|
|
|
end
|
|
|
|
|
|
|
|
def test_instrumentation_stale
|
2022-01-13 17:05:07 +08:00
|
|
|
file_path = Help.set_file("a.rb", "a = a = 3", 100)
|
|
|
|
load(file_path)
|
|
|
|
file_path = Help.set_file("a.rb", "a = a = 4", 101)
|
2021-02-01 18:41:46 +08:00
|
|
|
|
|
|
|
calls = []
|
|
|
|
Bootsnap.instrumentation = ->(event, path) { calls << [event, path] }
|
|
|
|
|
2022-01-13 17:05:07 +08:00
|
|
|
load(file_path)
|
2021-02-01 18:41:46 +08:00
|
|
|
|
2022-01-13 17:05:07 +08:00
|
|
|
assert_equal [[:stale, "a.rb"]], calls
|
2021-02-01 18:41:46 +08:00
|
|
|
end
|
2017-01-19 00:28:16 +08:00
|
|
|
end
|