From 3db039a183cd836e2783bccd6e8a9d04d4b83b52 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 14 Dec 2020 16:43:29 +0100 Subject: [PATCH] improve serialization of lazy elements --- lib/serialization/BinaryMiddleware.js | 137 +++++++++++++++++++------- test/BinaryMiddleware.unittest.js | 42 ++++++-- 2 files changed, 136 insertions(+), 43 deletions(-) diff --git a/lib/serialization/BinaryMiddleware.js b/lib/serialization/BinaryMiddleware.js index 01af2d10a..438645307 100644 --- a/lib/serialization/BinaryMiddleware.js +++ b/lib/serialization/BinaryMiddleware.js @@ -216,37 +216,58 @@ class BinaryMiddleware extends SerializerMiddleware { case "function": { if (!SerializerMiddleware.isLazy(thing)) throw new Error("Unexpected function " + thing); - /** @type {SerializedType[0]} */ - const serializedData = SerializerMiddleware.getLazySerializedValue( + /** @type {SerializedType | (() => SerializedType)} */ + let serializedData = SerializerMiddleware.getLazySerializedValue( thing ); - if (serializedData !== undefined) { - if (typeof serializedData === "function") { - flush(); - buffers.push(serializedData); + if (serializedData === undefined) { + if (SerializerMiddleware.isLazy(thing, this)) { + const data = this._serialize(thing(), context); + SerializerMiddleware.setLazySerializedValue(thing, data); + serializedData = data; } else { - serializeData(serializedData); - allocate(5); - writeU8(LAZY_HEADER); - writeU32(serializedData.length); + serializedData = SerializerMiddleware.serializeLazy( + thing, + data => this._serialize(data, context) + ); } - } else if (SerializerMiddleware.isLazy(thing, this)) { - /** @type {SerializedType} */ - const data = BinaryMiddleware.optimizeSerializedData( - this._serialize(thing(), context) - ); - SerializerMiddleware.setLazySerializedValue(thing, data); - serializeData(data); - allocate(5); - writeU8(LAZY_HEADER); - writeU32(data.length); - } else { + } + if (typeof serializedData === "function") { flush(); - buffers.push( - SerializerMiddleware.serializeLazy(thing, data => - this._serialize(data, context) - ) - ); + buffers.push(serializedData); + } else { + const lengths = []; + for (const item of serializedData) { + let last; + if (typeof item === "function") { + lengths.push(0); + } else if (item.length === 0) { + // ignore + } else if ( + lengths.length > 0 && + (last = lengths[lengths.length - 1]) !== 0 + ) { + const remaining = 0xffffffff - last; + if (remaining >= item.length) { + lengths[lengths.length - 1] += item.length; + } else { + lengths.push(item.length - remaining); + lengths[lengths.length - 2] = 0xffffffff; + } + } else { + lengths.push(item.length); + } + } + allocate(5 + lengths.length * 4); + writeU8(LAZY_HEADER); + writeU32(lengths.length); + for (const l of lengths) { + writeU32(l); + } + for (const item of serializedData) { + flush(); + buffers.push(item); + } } break; } @@ -519,6 +540,31 @@ class BinaryMiddleware extends SerializerMiddleware { checkOverflow(); return res; }; + /** + * Reads up to n bytes + * @param {number} n amount of bytes to read + * @returns {Buffer} buffer with bytes + */ + const readUpTo = n => { + if (!currentIsBuffer) { + throw new Error( + currentBuffer === null + ? "Unexpected end of stream" + : "Unexpected lazy element in stream" + ); + } + const rem = currentBuffer.length - currentPosition; + if (rem < n) { + n = rem; + } + const res = /** @type {Buffer} */ (currentBuffer).slice( + currentPosition, + currentPosition + n + ); + currentPosition += n; + checkOverflow(); + return res; + }; const readU8 = () => { if (!currentIsBuffer) { throw new Error( @@ -554,15 +600,32 @@ class BinaryMiddleware extends SerializerMiddleware { case LAZY_HEADER: return () => { const count = readU32(); - const start = result.length - count; - const data = /** @type {SerializedType} */ (result.slice(start)); - result.length = start; + const lengths = Array.from({ length: count }).map(() => readU32()); + const content = []; + for (let l of lengths) { + if (l === 0) { + if (typeof currentBuffer !== "function") { + throw new Error("Unexpected non-lazy element in stream"); + } + content.push(currentBuffer); + currentDataItem++; + currentBuffer = + currentDataItem < data.length ? data[currentDataItem] : null; + currentIsBuffer = Buffer.isBuffer(currentBuffer); + } else { + do { + const buf = readUpTo(l); + l -= buf.length; + content.push(buf); + } while (l > 0); + } + } result.push( SerializerMiddleware.createLazy( - memorize(() => this._deserialize(data, context)), + memorize(() => this._deserialize(content, context)), this, undefined, - data + content ) ); }; @@ -587,7 +650,7 @@ class BinaryMiddleware extends SerializerMiddleware { return () => result.push(null, false); case NULL_AND_I8_HEADER: return () => { - if (currentBuffer) { + if (currentIsBuffer) { result.push( null, /** @type {Buffer} */ (currentBuffer).readInt8(currentPosition) @@ -675,7 +738,7 @@ class BinaryMiddleware extends SerializerMiddleware { return () => result.push(""); case SHORT_STRING_HEADER | 1: return () => { - if (currentBuffer) { + if (currentIsBuffer) { result.push( currentBuffer.toString( "latin1", @@ -691,7 +754,7 @@ class BinaryMiddleware extends SerializerMiddleware { }; case I8_HEADER: return () => { - if (currentBuffer) { + if (currentIsBuffer) { result.push( /** @type {Buffer} */ (currentBuffer).readInt8(currentPosition) ); @@ -807,10 +870,10 @@ class BinaryMiddleware extends SerializerMiddleware { currentBuffer = currentDataItem < data.length ? data[currentDataItem] : null; currentIsBuffer = Buffer.isBuffer(currentBuffer); - continue; + } else { + const header = readU8(); + dispatchTable[header](); } - const header = readU8(); - dispatchTable[header](); } return result; } diff --git a/test/BinaryMiddleware.unittest.js b/test/BinaryMiddleware.unittest.js index 5445da457..1b548c12c 100644 --- a/test/BinaryMiddleware.unittest.js +++ b/test/BinaryMiddleware.unittest.js @@ -1,4 +1,5 @@ const BinaryMiddleware = require("../lib/serialization/BinaryMiddleware"); +const SerializerMiddleware = require("../lib/serialization/SerializerMiddleware"); const cont = (base, count) => { const result = []; @@ -8,9 +9,22 @@ const cont = (base, count) => { return result; }; +const mw = new BinaryMiddleware(); +const other = { other: true }; + +const resolveLazy = item => { + if (SerializerMiddleware.isLazy(item)) { + // console.log("resolve lazy", item); + const data = item(); + // console.log("resolve lazy done", data); + if (Array.isArray(data)) return { resolvesTo: data.map(resolveLazy) }; + return { resolvesTo: resolveLazy(data) }; + } + return item; +}; + describe("BinaryMiddleware", () => { const items = [ - undefined, true, false, null, @@ -26,12 +40,29 @@ describe("BinaryMiddleware", () => { -1, -11, -0x100, - -1.25 + -1.25, + SerializerMiddleware.createLazy([5], other), + SerializerMiddleware.createLazy( + [SerializerMiddleware.createLazy([5], other)], + mw + ), + SerializerMiddleware.createLazy( + [ + 1, + SerializerMiddleware.createLazy([2], mw), + SerializerMiddleware.createLazy([5], other), + 4 + ], + mw + ) ]; + items.push(SerializerMiddleware.createLazy(items.slice(), mw)); + items.push(SerializerMiddleware.createLazy(items.slice(), other)); + items.push(undefined); const cases = [ ...items.map(item => [item]), - [true, true], + [(true, true)], [false, true], [true, false], [false, false], @@ -71,12 +102,11 @@ describe("BinaryMiddleware", () => { x => x !== undefined ); if (data.length === 0) continue; - const key = JSON.stringify(data); + const key = JSON.stringify(data.map(resolveLazy)); it(`should serialize ${key} (${data.length}) correctly`, () => { - const mw = new BinaryMiddleware(); const serialized = mw.serialize(data, {}); const newData = mw.deserialize(serialized, {}); - expect(newData).toEqual(data); + expect(newData.map(resolveLazy)).toEqual(data.map(resolveLazy)); }); } }