root
/
webpack
mirror of https://github.com/webpack/webpack.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #13662 from webpack/perf/avoid-splitting-buffer 

reduce amount of buffer splitting for large cache files
This commit is contained in:
Tobias Koppers 2021-06-29 16:26:24 +02:00 committed by GitHub
parent d2640d347e 563a28dd0f
commit 74a16d0807
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 148 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
lib/TemplatedPathPlugin.js
View File

@ -116,32 +116,30 @@ const replacePathVariables = (path, data, assetInfo) => {
// [path] - /some/path/
// [name] - file
// [ext] - .js
if (data.filename) {
if (typeof data.filename === "string") {
const { path: file, query, fragment } = parseResource(data.filename);
if (typeof data.filename === "string") {
const { path: file, query, fragment } = parseResource(data.filename);
const ext = extname(file);
const base = basename(file);
const name = base.slice(0, base.length - ext.length);
const path = file.slice(0, file.length - base.length);
const ext = extname(file);
const base = basename(file);
const name = base.slice(0, base.length - ext.length);
const path = file.slice(0, file.length - base.length);
replacements.set("file", replacer(file));
replacements.set("query", replacer(query, true));
replacements.set("fragment", replacer(fragment, true));
replacements.set("path", replacer(path, true));
replacements.set("base", replacer(base));
replacements.set("name", replacer(name));
replacements.set("ext", replacer(ext, true));
// Legacy
replacements.set(
"filebase",
deprecated(
replacer(base),
"[filebase] is now [base]",
"DEP_WEBPACK_TEMPLATE_PATH_PLUGIN_REPLACE_PATH_VARIABLES_FILENAME"
)
);
}
replacements.set("file", replacer(file));
replacements.set("query", replacer(query, true));
replacements.set("fragment", replacer(fragment, true));
replacements.set("path", replacer(path, true));
replacements.set("base", replacer(base));
replacements.set("name", replacer(name));
replacements.set("ext", replacer(ext, true));
// Legacy
replacements.set(
"filebase",
deprecated(
replacer(base),
"[filebase] is now [base]",
"DEP_WEBPACK_TEMPLATE_PATH_PLUGIN_REPLACE_PATH_VARIABLES_FILENAME"
)
);
}
// Compilation context

38
lib/serialization/BinaryMiddleware.js
View File

@ -165,12 +165,23 @@ class BinaryMiddleware extends SerializerMiddleware {
};
const flush = () => {
if (currentBuffer !== null) {
buffers.push(currentBuffer.slice(0, currentPosition));
buffers.push(
Buffer.from(
currentBuffer.buffer,
currentBuffer.byteOffset,
currentPosition
)
);
if (
!leftOverBuffer ||
leftOverBuffer.length < currentBuffer.length - currentPosition
)
leftOverBuffer = currentBuffer.slice(currentPosition);
) {
leftOverBuffer = Buffer.from(
currentBuffer.buffer,
currentBuffer.byteOffset + currentPosition,
currentBuffer.byteLength - currentPosition
);
}
currentBuffer = null;
buffersTotalLength += currentPosition;
currentPosition = 0;
@ -554,10 +565,8 @@ class BinaryMiddleware extends SerializerMiddleware {
if (rem < n) {
return Buffer.concat([read(rem), read(n - rem)]);
}
const res = /** @type {Buffer} */ (currentBuffer).slice(
currentPosition,
currentPosition + n
);
const b = /** @type {Buffer} */ (currentBuffer);
const res = Buffer.from(b.buffer, b.byteOffset + currentPosition, n);
currentPosition += n;
checkOverflow();
return res;
@ -579,10 +588,8 @@ class BinaryMiddleware extends SerializerMiddleware {
if (rem < n) {
n = rem;
}
const res = /** @type {Buffer} */ (currentBuffer).slice(
currentPosition,
currentPosition + n
);
const b = /** @type {Buffer} */ (currentBuffer);
const res = Buffer.from(b.buffer, b.byteOffset + currentPosition, n);
currentPosition += n;
checkOverflow();
return res;
@ -735,7 +742,7 @@ class BinaryMiddleware extends SerializerMiddleware {
case STRING_HEADER:
return () => {
const len = readU32();
if (isInCurrentBuffer(len)) {
if (isInCurrentBuffer(len) && currentPosition + len < 0x7fffffff) {
result.push(
currentBuffer.toString(
undefined,
@ -753,7 +760,7 @@ class BinaryMiddleware extends SerializerMiddleware {
return () => result.push("");
case SHORT_STRING_HEADER | 1:
return () => {
if (currentIsBuffer) {
if (currentIsBuffer && currentPosition < 0x7ffffffe) {
result.push(
currentBuffer.toString(
"latin1",
@ -785,7 +792,10 @@ class BinaryMiddleware extends SerializerMiddleware {
} else if ((header & SHORT_STRING_HEADER) === SHORT_STRING_HEADER) {
const len = header & SHORT_STRING_LENGTH_MASK;
return () => {
if (isInCurrentBuffer(len)) {
if (
isInCurrentBuffer(len) &&
currentPosition + len < 0x7fffffff
) {
result.push(
currentBuffer.toString(
"latin1",

24
lib/serialization/FileMiddleware.js
View File

@ -280,8 +280,16 @@ const deserialize = async (middleware, name, readFile) => {
}
const sectionCount = readUInt32LE();
const lengths = [];
let lastLengthPositive = false;
for (let i = 0; i < sectionCount; i++) {
lengths.push(readInt32LE());
const value = readInt32LE();
const valuePositive = value >= 0;
if (lastLengthPositive && valuePositive) {
lengths[lengths.length - 1] += value;
} else {
lengths.push(value);
lastLengthPositive = valuePositive;
}
}
const result = [];
for (let length of lengths) {
@ -316,13 +324,15 @@ const deserialize = async (middleware, name, readFile) => {
contentPosition += length;
length = 0;
} else {
const l = contentItemLength - contentPosition;
result.push(
Buffer.from(
contentItem.buffer,
contentItem.byteOffset + contentPosition
contentItem.byteOffset + contentPosition,
l
)
);
length -= contentItemLength - contentPosition;
length -= l;
contentPosition = contentItemLength;
}
} else {
@ -331,7 +341,9 @@ const deserialize = async (middleware, name, readFile) => {
length -= contentItemLength;
contentPosition = contentItemLength;
} else {
result.push(contentItem.slice(0, length));
result.push(
Buffer.from(contentItem.buffer, contentItem.byteOffset, length)
);
contentPosition += length;
length = 0;
}
@ -343,7 +355,9 @@ const deserialize = async (middleware, name, readFile) => {
length -= contentItemLength;
contentPosition = contentItemLength;
} else {
result.push(contentItem.slice(0, length));
result.push(
Buffer.from(contentItem.buffer, contentItem.byteOffset, length)
);
contentPosition += length;
length = 0;
}

75
test/TestCases.template.js
View File

@ -95,6 +95,11 @@ const describeCases = config => {
category.name,
testName
);
let testConfig = {};
const testConfigPath = path.join(testDirectory, "test.config.js");
if (fs.existsSync(testConfigPath)) {
testConfig = require(testConfigPath);
}
const options = {
context: casesPath,
entry: "./" + category.name + "/" + testName + "/",
@ -196,34 +201,42 @@ const describeCases = config => {
rimraf(cacheDirectory, done);
});
if (config.cache) {
it(`${testName} should pre-compile to fill disk cache (1st)`, done => {
const oldPath = options.output.path;
options.output.path = path.join(
options.output.path,
"cache1"
);
const deprecationTracker = deprecationTracking.start();
webpack(options, err => {
deprecationTracker();
options.output.path = oldPath;
if (err) return done(err);
done();
});
}, 60000);
it(`${testName} should pre-compile to fill disk cache (2nd)`, done => {
const oldPath = options.output.path;
options.output.path = path.join(
options.output.path,
"cache2"
);
const deprecationTracker = deprecationTracking.start();
webpack(options, err => {
deprecationTracker();
options.output.path = oldPath;
if (err) return done(err);
done();
});
}, 10000);
it(
`${testName} should pre-compile to fill disk cache (1st)`,
done => {
const oldPath = options.output.path;
options.output.path = path.join(
options.output.path,
"cache1"
);
const deprecationTracker = deprecationTracking.start();
webpack(options, err => {
deprecationTracker();
options.output.path = oldPath;
if (err) return done(err);
done();
});
},
testConfig.timeout || 60000
);
it(
`${testName} should pre-compile to fill disk cache (2nd)`,
done => {
const oldPath = options.output.path;
options.output.path = path.join(
options.output.path,
"cache2"
);
const deprecationTracker = deprecationTracking.start();
webpack(options, err => {
deprecationTracker();
options.output.path = oldPath;
if (err) return done(err);
done();
});
},
testConfig.cachedTimeout || testConfig.timeout || 10000
);
}
it(
testName + " should compile",
@ -303,7 +316,9 @@ const describeCases = config => {
run();
}
},
config.cache ? 20000 : 60000
testConfig.cachedTimeout ||
testConfig.timeout ||
(config.cache ? 20000 : 60000)
);
it(
@ -415,7 +430,7 @@ const describeCases = config => {
const { it: _it, getNumberOfTests } = createLazyTestEnv(
jasmine.getEnv(),
10000
testConfig.timeout || 10000
);
});
});

6
test/cases/large/big-assets/generate-big-asset-loader.js Normal file
View File

@ -0,0 +1,6 @@
/** @type {import("../../../../").RawLoaderDefinition<{ size: string }>} */
module.exports = function () {
const options = this.getOptions();
return Buffer.alloc(+options.size);
};
module.exports.raw = true;

26
test/cases/large/big-assets/index.js Normal file
View File

@ -0,0 +1,26 @@
it("should compile fine", () => {
const a = new URL(
"./generate-big-asset-loader.js?size=100000000!",
import.meta.url
);
const b = new URL(
"./generate-big-asset-loader.js?size=200000000!",
import.meta.url
);
const c = new URL(
"./generate-big-asset-loader.js?size=300000000!",
import.meta.url
);
const d = new URL(
"./generate-big-asset-loader.js?size=400000000!",
import.meta.url
);
const e = new URL(
"./generate-big-asset-loader.js?size=500000000!",
import.meta.url
);
const f = new URL(
"./generate-big-asset-loader.js?size=600000000!",
import.meta.url
);
});

3
test/cases/large/big-assets/test.config.js Normal file
View File

@ -0,0 +1,3 @@
module.exports = {
timeout: 120000
};

3
test/cases/large/big-assets/test.filter.js Normal file
View File

@ -0,0 +1,3 @@
module.exports = function (config) {
return !process.env.CI;
};