improve string (de)serialization

lib/serialization/BinaryMiddleware.js

@@ -35,7 +35,7 @@ BooleansSection -> TrueHeaderByte | FalseHeaderByte | BooleansSectionHeaderByte
 F64NumbersSection -> F64NumbersSectionHeaderByte f64*
 I32NumbersSection -> I32NumbersSectionHeaderByte i32*
 I8NumbersSection -> I8NumbersSectionHeaderByte i8*
-ShortStringSection -> ShortStringSectionHeaderByte utf8-byte*
+ShortStringSection -> ShortStringSectionHeaderByte ascii-byte*
 StringSection -> StringSectionHeaderByte i32:length utf8-byte*
 BufferSection -> BufferSectionHeaderByte i32:length byte*
 NopSection --> NopSectionHeaderByte
@@ -250,15 +250,16 @@ class BinaryMiddleware extends SerializerMiddleware {
 					}
 					case "string": {
 						const len = Buffer.byteLength(thing);
-						if (len >= 128) {
+						if (len >= 128 || len !== thing.length) {
 							allocate(len + HEADER_SIZE + I32_SIZE);
 							writeU8(STRING_HEADER);
 							writeU32(len);
+							currentBuffer.write(thing, currentPosition);
 						} else {
 							allocate(len + HEADER_SIZE);
 							writeU8(SHORT_STRING_HEADER | len);
+							currentBuffer.write(thing, currentPosition, "latin1");
 						}
-						currentBuffer.write(thing, currentPosition);
 						currentPosition += len;
 						break;
 					}
@@ -670,8 +671,20 @@ class BinaryMiddleware extends SerializerMiddleware {
 				}
 				case STRING_HEADER: {
 					const len = readU32();
-					const buf = read(len);
-					result.push(buf.toString());
+					if (isInCurrentBuffer(len)) {
+						result.push(
+							currentBuffer.toString(
+								undefined,
+								currentPosition,
+								currentPosition + len
+							)
+						);
+						currentPosition += len;
+						checkOverflow();
+					} else {
+						const buf = read(len);
+						result.push(buf.toString());
+					}
 					break;
 				}
 				default:
@@ -681,9 +694,19 @@ class BinaryMiddleware extends SerializerMiddleware {
 						const len = header & SHORT_STRING_LENGTH_MASK;
 						if (len === 0) {
 							result.push("");
+						} else if (isInCurrentBuffer(len)) {
+							result.push(
+								currentBuffer.toString(
+									"latin1",
+									currentPosition,
+									currentPosition + len
+								)
+							);
+							currentPosition += len;
+							checkOverflow();
 						} else {
 							const buf = read(len);
-							result.push(buf.toString());
+							result.push(buf.toString("latin1"));
 						}
 					} else if ((header & NUMBERS_HEADER_MASK) === F64_HEADER) {
 						const len = (header & NUMBERS_COUNT_MASK) + 1;

test/BinaryMiddleware.unittest.js

@@ -9,20 +9,28 @@ const cont = (base, count) => {
 };
 
 describe("BinaryMiddleware", () => {
+	const items = [
+		undefined,
+		true,
+		false,
+		null,
+		"",
+		"hi",
+		"hi".repeat(200),
+		"😀",
+		"😀".repeat(200),
+		Buffer.from("hello"),
+		1,
+		11,
+		0x100,
+		-1,
+		-11,
+		-0x100,
+		-1.25
+	];
+
 	const cases = [
-		[true],
-		[false],
-		[null],
-		[""],
-		["hi"],
-		[Buffer.from("hello")],
-		[1],
-		[11],
-		[0x100],
-		[-1],
-		[-11],
-		[-0x100],
-		[-1.25],
+		...items.map(item => [item]),
 		[true, true],
 		[false, true],
 		[true, false],
@@ -44,24 +52,16 @@ describe("BinaryMiddleware", () => {
 		cont([false, true, false, true], 133),
 		cont([false, true, false, true], 134),
 		cont([false, true, false, true], 135),
-		cont([true], 135)
-	];
-
-	const items = [
-		undefined,
-		true,
-		false,
-		null,
-		"",
-		"hi",
-		Buffer.from("hello"),
-		1,
-		11,
-		0x100,
-		-1,
-		-11,
-		-0x100,
-		-1.25
+		cont([true], 135),
+		[null],
+		[null, null],
+		[null, null, null],
+		cont([null], 4),
+		cont([null], 100),
+		cont([null], 300),
+		cont([-20], 20),
+		cont([400], 20),
+		cont([5.5], 20)
 	];
 
 	for (const caseData of cases) {
@@ -70,6 +70,7 @@ describe("BinaryMiddleware", () => {
 				const data = [prepend, ...caseData, append].filter(
 					x => x !== undefined
 				);
+				if (data.length === 0) continue;
 				const key = JSON.stringify(data);
 				it(`should serialize ${key} (${data.length}) correctly`, () => {
 					const mw = new BinaryMiddleware();