improve serialization of lazy elements

lib/serialization/BinaryMiddleware.js

@@ -216,37 +216,58 @@ class BinaryMiddleware extends SerializerMiddleware {
 					case "function": {
 						if (!SerializerMiddleware.isLazy(thing))
 							throw new Error("Unexpected function " + thing);
-						/** @type {SerializedType[0]} */
+						/** @type {SerializedType | (() => SerializedType)} */
-						const serializedData = SerializerMiddleware.getLazySerializedValue(
+						let serializedData = SerializerMiddleware.getLazySerializedValue(
 							thing
 						);
-						if (serializedData !== undefined) {
+						if (serializedData === undefined) {
-							if (typeof serializedData === "function") {
+							if (SerializerMiddleware.isLazy(thing, this)) {
-								flush();
+								const data = this._serialize(thing(), context);
-								buffers.push(serializedData);
+								SerializerMiddleware.setLazySerializedValue(thing, data);
+								serializedData = data;
 							} else {
-								serializeData(serializedData);
+								serializedData = SerializerMiddleware.serializeLazy(
-								allocate(5);
+									thing,
-								writeU8(LAZY_HEADER);
+									data => this._serialize(data, context)
-								writeU32(serializedData.length);
+								);
 							}
-						} else if (SerializerMiddleware.isLazy(thing, this)) {
+						}
-							/** @type {SerializedType} */
+						if (typeof serializedData === "function") {
-							const data = BinaryMiddleware.optimizeSerializedData(
-								this._serialize(thing(), context)
-							);
-							SerializerMiddleware.setLazySerializedValue(thing, data);
-							serializeData(data);
-							allocate(5);
-							writeU8(LAZY_HEADER);
-							writeU32(data.length);
-						} else {
 							flush();
-							buffers.push(
+							buffers.push(serializedData);
-								SerializerMiddleware.serializeLazy(thing, data =>
+						} else {
-									this._serialize(data, context)
+							const lengths = [];
-								)
+							for (const item of serializedData) {
-							);
+								let last;
+								if (typeof item === "function") {
+									lengths.push(0);
+								} else if (item.length === 0) {
+									// ignore
+								} else if (
+									lengths.length > 0 &&
+									(last = lengths[lengths.length - 1]) !== 0
+								) {
+									const remaining = 0xffffffff - last;
+									if (remaining >= item.length) {
+										lengths[lengths.length - 1] += item.length;
+									} else {
+										lengths.push(item.length - remaining);
+										lengths[lengths.length - 2] = 0xffffffff;
+									}
+								} else {
+									lengths.push(item.length);
+								}
+							}
+							allocate(5 + lengths.length * 4);
+							writeU8(LAZY_HEADER);
+							writeU32(lengths.length);
+							for (const l of lengths) {
+								writeU32(l);
+							}
+							for (const item of serializedData) {
+								flush();
+								buffers.push(item);
+							}
 						}
 						break;
 					}
@@ -519,6 +540,31 @@ class BinaryMiddleware extends SerializerMiddleware {
 			checkOverflow();
 			return res;
 		};
+		/**
+		 * Reads up to n bytes
+		 * @param {number} n amount of bytes to read
+		 * @returns {Buffer} buffer with bytes
+		 */
+		const readUpTo = n => {
+			if (!currentIsBuffer) {
+				throw new Error(
+					currentBuffer === null
+						? "Unexpected end of stream"
+						: "Unexpected lazy element in stream"
+				);
+			}
+			const rem = currentBuffer.length - currentPosition;
+			if (rem < n) {
+				n = rem;
+			}
+			const res = /** @type {Buffer} */ (currentBuffer).slice(
+				currentPosition,
+				currentPosition + n
+			);
+			currentPosition += n;
+			checkOverflow();
+			return res;
+		};
 		const readU8 = () => {
 			if (!currentIsBuffer) {
 				throw new Error(
@@ -554,15 +600,32 @@ class BinaryMiddleware extends SerializerMiddleware {
 				case LAZY_HEADER:
 					return () => {
 						const count = readU32();
-						const start = result.length - count;
+						const lengths = Array.from({ length: count }).map(() => readU32());
-						const data = /** @type {SerializedType} */ (result.slice(start));
+						const content = [];
-						result.length = start;
+						for (let l of lengths) {
+							if (l === 0) {
+								if (typeof currentBuffer !== "function") {
+									throw new Error("Unexpected non-lazy element in stream");
+								}
+								content.push(currentBuffer);
+								currentDataItem++;
+								currentBuffer =
+									currentDataItem < data.length ? data[currentDataItem] : null;
+								currentIsBuffer = Buffer.isBuffer(currentBuffer);
+							} else {
+								do {
+									const buf = readUpTo(l);
+									l -= buf.length;
+									content.push(buf);
+								} while (l > 0);
+							}
+						}
 						result.push(
 							SerializerMiddleware.createLazy(
-								memorize(() => this._deserialize(data, context)),
+								memorize(() => this._deserialize(content, context)),
 								this,
 								undefined,
-								data
+								content
 							)
 						);
 					};
@@ -587,7 +650,7 @@ class BinaryMiddleware extends SerializerMiddleware {
 					return () => result.push(null, false);
 				case NULL_AND_I8_HEADER:
 					return () => {
-						if (currentBuffer) {
+						if (currentIsBuffer) {
 							result.push(
 								null,
 								/** @type {Buffer} */ (currentBuffer).readInt8(currentPosition)
@@ -675,7 +738,7 @@ class BinaryMiddleware extends SerializerMiddleware {
 					return () => result.push("");
 				case SHORT_STRING_HEADER | 1:
 					return () => {
-						if (currentBuffer) {
+						if (currentIsBuffer) {
 							result.push(
 								currentBuffer.toString(
 									"latin1",
@@ -691,7 +754,7 @@ class BinaryMiddleware extends SerializerMiddleware {
 					};
 				case I8_HEADER:
 					return () => {
-						if (currentBuffer) {
+						if (currentIsBuffer) {
 							result.push(
 								/** @type {Buffer} */ (currentBuffer).readInt8(currentPosition)
 							);
@@ -807,10 +870,10 @@ class BinaryMiddleware extends SerializerMiddleware {
 				currentBuffer =
 					currentDataItem < data.length ? data[currentDataItem] : null;
 				currentIsBuffer = Buffer.isBuffer(currentBuffer);
-				continue;
+			} else {
+				const header = readU8();
+				dispatchTable[header]();
 			}
-			const header = readU8();
-			dispatchTable[header]();
 		}
 		return result;
 	}

test/BinaryMiddleware.unittest.js

@@ -1,4 +1,5 @@
 const BinaryMiddleware = require("../lib/serialization/BinaryMiddleware");
+const SerializerMiddleware = require("../lib/serialization/SerializerMiddleware");
 
 const cont = (base, count) => {
 	const result = [];
@@ -8,9 +9,22 @@ const cont = (base, count) => {
 	return result;
 };
 
+const mw = new BinaryMiddleware();
+const other = { other: true };
+
+const resolveLazy = item => {
+	if (SerializerMiddleware.isLazy(item)) {
+		// console.log("resolve lazy", item);
+		const data = item();
+		// console.log("resolve lazy done", data);
+		if (Array.isArray(data)) return { resolvesTo: data.map(resolveLazy) };
+		return { resolvesTo: resolveLazy(data) };
+	}
+	return item;
+};
+
 describe("BinaryMiddleware", () => {
 	const items = [
-		undefined,
 		true,
 		false,
 		null,
@@ -26,12 +40,29 @@ describe("BinaryMiddleware", () => {
 		-1,
 		-11,
 		-0x100,
-		-1.25
+		-1.25,
+		SerializerMiddleware.createLazy([5], other),
+		SerializerMiddleware.createLazy(
+			[SerializerMiddleware.createLazy([5], other)],
+			mw
+		),
+		SerializerMiddleware.createLazy(
+			[
+				1,
+				SerializerMiddleware.createLazy([2], mw),
+				SerializerMiddleware.createLazy([5], other),
+				4
+			],
+			mw
+		)
 	];
+	items.push(SerializerMiddleware.createLazy(items.slice(), mw));
+	items.push(SerializerMiddleware.createLazy(items.slice(), other));
+	items.push(undefined);
 
 	const cases = [
 		...items.map(item => [item]),
-		[true, true],
+		[(true, true)],
 		[false, true],
 		[true, false],
 		[false, false],
@@ -71,12 +102,11 @@ describe("BinaryMiddleware", () => {
 					x => x !== undefined
 				);
 				if (data.length === 0) continue;
-				const key = JSON.stringify(data);
+				const key = JSON.stringify(data.map(resolveLazy));
 				it(`should serialize ${key} (${data.length}) correctly`, () => {
-					const mw = new BinaryMiddleware();
 					const serialized = mw.serialize(data, {});
 					const newData = mw.deserialize(serialized, {});
-					expect(newData).toEqual(data);
+					expect(newData.map(resolveLazy)).toEqual(data.map(resolveLazy));
 				});
 			}
 		}