mirror of https://github.com/webpack/webpack.git
improve serialization of lazy elements
This commit is contained in:
Tobias Koppers
parent
9563338d90
commit
3db039a183
|
|
@ -216,37 +216,58 @@ class BinaryMiddleware extends SerializerMiddleware {
|
|||
case "function": {
|
||||
if (!SerializerMiddleware.isLazy(thing))
|
||||
throw new Error("Unexpected function " + thing);
|
||||
/** @type {SerializedType[0]} */
|
||||
const serializedData = SerializerMiddleware.getLazySerializedValue(
|
||||
/** @type {SerializedType | (() => SerializedType)} */
|
||||
let serializedData = SerializerMiddleware.getLazySerializedValue(
|
||||
thing
|
||||
);
|
||||
if (serializedData !== undefined) {
|
||||
if (typeof serializedData === "function") {
|
||||
flush();
|
||||
buffers.push(serializedData);
|
||||
if (serializedData === undefined) {
|
||||
if (SerializerMiddleware.isLazy(thing, this)) {
|
||||
const data = this._serialize(thing(), context);
|
||||
SerializerMiddleware.setLazySerializedValue(thing, data);
|
||||
serializedData = data;
|
||||
} else {
|
||||
serializeData(serializedData);
|
||||
allocate(5);
|
||||
writeU8(LAZY_HEADER);
|
||||
writeU32(serializedData.length);
|
||||
serializedData = SerializerMiddleware.serializeLazy(
|
||||
thing,
|
||||
data => this._serialize(data, context)
|
||||
);
|
||||
}
|
||||
} else if (SerializerMiddleware.isLazy(thing, this)) {
|
||||
/** @type {SerializedType} */
|
||||
const data = BinaryMiddleware.optimizeSerializedData(
|
||||
this._serialize(thing(), context)
|
||||
);
|
||||
SerializerMiddleware.setLazySerializedValue(thing, data);
|
||||
serializeData(data);
|
||||
allocate(5);
|
||||
writeU8(LAZY_HEADER);
|
||||
writeU32(data.length);
|
||||
} else {
|
||||
}
|
||||
if (typeof serializedData === "function") {
|
||||
flush();
|
||||
buffers.push(
|
||||
SerializerMiddleware.serializeLazy(thing, data =>
|
||||
this._serialize(data, context)
|
||||
)
|
||||
);
|
||||
buffers.push(serializedData);
|
||||
} else {
|
||||
const lengths = [];
|
||||
for (const item of serializedData) {
|
||||
let last;
|
||||
if (typeof item === "function") {
|
||||
lengths.push(0);
|
||||
} else if (item.length === 0) {
|
||||
// ignore
|
||||
} else if (
|
||||
lengths.length > 0 &&
|
||||
(last = lengths[lengths.length - 1]) !== 0
|
||||
) {
|
||||
const remaining = 0xffffffff - last;
|
||||
if (remaining >= item.length) {
|
||||
lengths[lengths.length - 1] += item.length;
|
||||
} else {
|
||||
lengths.push(item.length - remaining);
|
||||
lengths[lengths.length - 2] = 0xffffffff;
|
||||
}
|
||||
} else {
|
||||
lengths.push(item.length);
|
||||
}
|
||||
}
|
||||
allocate(5 + lengths.length * 4);
|
||||
writeU8(LAZY_HEADER);
|
||||
writeU32(lengths.length);
|
||||
for (const l of lengths) {
|
||||
writeU32(l);
|
||||
}
|
||||
for (const item of serializedData) {
|
||||
flush();
|
||||
buffers.push(item);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
|
@ -519,6 +540,31 @@ class BinaryMiddleware extends SerializerMiddleware {
|
|||
checkOverflow();
|
||||
return res;
|
||||
};
|
||||
/**
|
||||
* Reads up to n bytes
|
||||
* @param {number} n amount of bytes to read
|
||||
* @returns {Buffer} buffer with bytes
|
||||
*/
|
||||
const readUpTo = n => {
|
||||
if (!currentIsBuffer) {
|
||||
throw new Error(
|
||||
currentBuffer === null
|
||||
? "Unexpected end of stream"
|
||||
: "Unexpected lazy element in stream"
|
||||
);
|
||||
}
|
||||
const rem = currentBuffer.length - currentPosition;
|
||||
if (rem < n) {
|
||||
n = rem;
|
||||
}
|
||||
const res = /** @type {Buffer} */ (currentBuffer).slice(
|
||||
currentPosition,
|
||||
currentPosition + n
|
||||
);
|
||||
currentPosition += n;
|
||||
checkOverflow();
|
||||
return res;
|
||||
};
|
||||
const readU8 = () => {
|
||||
if (!currentIsBuffer) {
|
||||
throw new Error(
|
||||
|
|
@ -554,15 +600,32 @@ class BinaryMiddleware extends SerializerMiddleware {
|
|||
case LAZY_HEADER:
|
||||
return () => {
|
||||
const count = readU32();
|
||||
const start = result.length - count;
|
||||
const data = /** @type {SerializedType} */ (result.slice(start));
|
||||
result.length = start;
|
||||
const lengths = Array.from({ length: count }).map(() => readU32());
|
||||
const content = [];
|
||||
for (let l of lengths) {
|
||||
if (l === 0) {
|
||||
if (typeof currentBuffer !== "function") {
|
||||
throw new Error("Unexpected non-lazy element in stream");
|
||||
}
|
||||
content.push(currentBuffer);
|
||||
currentDataItem++;
|
||||
currentBuffer =
|
||||
currentDataItem < data.length ? data[currentDataItem] : null;
|
||||
currentIsBuffer = Buffer.isBuffer(currentBuffer);
|
||||
} else {
|
||||
do {
|
||||
const buf = readUpTo(l);
|
||||
l -= buf.length;
|
||||
content.push(buf);
|
||||
} while (l > 0);
|
||||
}
|
||||
}
|
||||
result.push(
|
||||
SerializerMiddleware.createLazy(
|
||||
memorize(() => this._deserialize(data, context)),
|
||||
memorize(() => this._deserialize(content, context)),
|
||||
this,
|
||||
undefined,
|
||||
data
|
||||
content
|
||||
)
|
||||
);
|
||||
};
|
||||
|
|
@ -587,7 +650,7 @@ class BinaryMiddleware extends SerializerMiddleware {
|
|||
return () => result.push(null, false);
|
||||
case NULL_AND_I8_HEADER:
|
||||
return () => {
|
||||
if (currentBuffer) {
|
||||
if (currentIsBuffer) {
|
||||
result.push(
|
||||
null,
|
||||
/** @type {Buffer} */ (currentBuffer).readInt8(currentPosition)
|
||||
|
|
@ -675,7 +738,7 @@ class BinaryMiddleware extends SerializerMiddleware {
|
|||
return () => result.push("");
|
||||
case SHORT_STRING_HEADER | 1:
|
||||
return () => {
|
||||
if (currentBuffer) {
|
||||
if (currentIsBuffer) {
|
||||
result.push(
|
||||
currentBuffer.toString(
|
||||
"latin1",
|
||||
|
|
@ -691,7 +754,7 @@ class BinaryMiddleware extends SerializerMiddleware {
|
|||
};
|
||||
case I8_HEADER:
|
||||
return () => {
|
||||
if (currentBuffer) {
|
||||
if (currentIsBuffer) {
|
||||
result.push(
|
||||
/** @type {Buffer} */ (currentBuffer).readInt8(currentPosition)
|
||||
);
|
||||
|
|
@ -807,10 +870,10 @@ class BinaryMiddleware extends SerializerMiddleware {
|
|||
currentBuffer =
|
||||
currentDataItem < data.length ? data[currentDataItem] : null;
|
||||
currentIsBuffer = Buffer.isBuffer(currentBuffer);
|
||||
continue;
|
||||
} else {
|
||||
const header = readU8();
|
||||
dispatchTable[header]();
|
||||
}
|
||||
const header = readU8();
|
||||
dispatchTable[header]();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
const BinaryMiddleware = require("../lib/serialization/BinaryMiddleware");
|
||||
const SerializerMiddleware = require("../lib/serialization/SerializerMiddleware");
|
||||
|
||||
const cont = (base, count) => {
|
||||
const result = [];
|
||||
|
|
@ -8,9 +9,22 @@ const cont = (base, count) => {
|
|||
return result;
|
||||
};
|
||||
|
||||
const mw = new BinaryMiddleware();
|
||||
const other = { other: true };
|
||||
|
||||
const resolveLazy = item => {
|
||||
if (SerializerMiddleware.isLazy(item)) {
|
||||
// console.log("resolve lazy", item);
|
||||
const data = item();
|
||||
// console.log("resolve lazy done", data);
|
||||
if (Array.isArray(data)) return { resolvesTo: data.map(resolveLazy) };
|
||||
return { resolvesTo: resolveLazy(data) };
|
||||
}
|
||||
return item;
|
||||
};
|
||||
|
||||
describe("BinaryMiddleware", () => {
|
||||
const items = [
|
||||
undefined,
|
||||
true,
|
||||
false,
|
||||
null,
|
||||
|
|
@ -26,12 +40,29 @@ describe("BinaryMiddleware", () => {
|
|||
-1,
|
||||
-11,
|
||||
-0x100,
|
||||
-1.25
|
||||
-1.25,
|
||||
SerializerMiddleware.createLazy([5], other),
|
||||
SerializerMiddleware.createLazy(
|
||||
[SerializerMiddleware.createLazy([5], other)],
|
||||
mw
|
||||
),
|
||||
SerializerMiddleware.createLazy(
|
||||
[
|
||||
1,
|
||||
SerializerMiddleware.createLazy([2], mw),
|
||||
SerializerMiddleware.createLazy([5], other),
|
||||
4
|
||||
],
|
||||
mw
|
||||
)
|
||||
];
|
||||
items.push(SerializerMiddleware.createLazy(items.slice(), mw));
|
||||
items.push(SerializerMiddleware.createLazy(items.slice(), other));
|
||||
items.push(undefined);
|
||||
|
||||
const cases = [
|
||||
...items.map(item => [item]),
|
||||
[true, true],
|
||||
[(true, true)],
|
||||
[false, true],
|
||||
[true, false],
|
||||
[false, false],
|
||||
|
|
@ -71,12 +102,11 @@ describe("BinaryMiddleware", () => {
|
|||
x => x !== undefined
|
||||
);
|
||||
if (data.length === 0) continue;
|
||||
const key = JSON.stringify(data);
|
||||
const key = JSON.stringify(data.map(resolveLazy));
|
||||
it(`should serialize ${key} (${data.length}) correctly`, () => {
|
||||
const mw = new BinaryMiddleware();
|
||||
const serialized = mw.serialize(data, {});
|
||||
const newData = mw.deserialize(serialized, {});
|
||||
expect(newData).toEqual(data);
|
||||
expect(newData.map(resolveLazy)).toEqual(data.map(resolveLazy));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue