Merge branch 'main' into dependabot/npm_and_yarn/is-ci-3.0.1

lib/util/hash/BatchedHash.js

@@ -36,7 +36,11 @@ class BatchedHash extends Hash {
 			this.string = undefined;
 		}
 		if (typeof data === "string") {
-			if (data.length < MAX_SHORT_STRING) {
+			if (
+				data.length < MAX_SHORT_STRING &&
+				// base64 encoding is not valid since it may contain padding chars
+				(!inputEncoding || !inputEncoding.startsWith("ba"))
+			) {
 				this.string = data;
 				this.encoding = inputEncoding;
 			} else {

lib/util/hash/wasm-hash.js

@@ -72,7 +72,7 @@ class WasmHash {
 						endPos += 2;
 					} else {
 						// bail-out for weird chars
-						endPos += mem.write(data.slice(endPos), endPos, encoding);
+						endPos += mem.write(data.slice(i), endPos, encoding);
 						break;
 					}
 				}

package.json

@@ -39,7 +39,7 @@
     "@babel/core": "^7.11.1",
     "@babel/preset-react": "^7.10.4",
     "@types/es-module-lexer": "^0.4.1",
-    "@types/jest": "^27.0.1",
+    "@types/jest": "^27.0.2",
     "@types/node": "^15.0.1",
     "assemblyscript": "^0.19.16",
     "babel-loader": "^8.1.0",
@@ -66,10 +66,10 @@
     "husky": "^6.0.0",
     "is-ci": "^3.0.0",
     "istanbul": "^0.4.5",
-    "jest": "^27.0.6",
-    "jest-circus": "^27.0.6",
-    "jest-cli": "^27.0.6",
-    "jest-diff": "^27.0.2",
+    "jest": "^27.3.1",
+    "jest-circus": "^27.3.1",
+    "jest-cli": "^27.3.1",
+    "jest-diff": "^27.3.1",
     "jest-junit": "^12.0.0",
     "json-loader": "^0.5.7",
     "json5": "^2.1.3",

test/WasmHashes.unittest.js

@@ -11,6 +11,15 @@ const wasmHashes = {
 			regExp: /^[0-9a-f]{16}$/
 		};
 	},
+	"xxhash64-createHash": () => {
+		const createXxHash = require("../lib/util/hash/xxhash64");
+		const createHash = require("../lib/util/createHash");
+		return {
+			createHash: () => createHash("xxhash64"),
+			createReferenceHash: createXxHash,
+			regExp: /^[0-9a-f]{16}$/
+		};
+	},
 	md4: () => {
 		const createMd4Hash = require("../lib/util/hash/md4");
 		return {
@@ -21,6 +30,15 @@ const wasmHashes = {
 					: createMd4Hash,
 			regExp: /^[0-9a-f]{32}$/
 		};
+	},
+	"md4-createHash": () => {
+		const createMd4Hash = require("../lib/util/hash/md4");
+		const createHash = require("../lib/util/createHash");
+		return {
+			createHash: () => createHash("md4"),
+			createReferenceHash: createMd4Hash,
+			regExp: /^[0-9a-f]{32}$/
+		};
 	}
 };
 
@@ -95,5 +113,46 @@ for (const name of Object.keys(wasmHashes)) {
 		test(`many updates 2`, sizes.slice().reverse());
 		test(`many updates 3`, sizes.concat(sizes.slice().reverse()));
 		test(`many updates 4`, sizes.slice().reverse().concat(sizes));
+
+		const unicodeTest = (name, codePoints) => {
+			it(name + " should hash unicode chars correctly", async () => {
+				const hash = createHash();
+				const reference = await createReferenceHash();
+				const str =
+					typeof codePoints === "string"
+						? codePoints
+						: String.fromCodePoint(...codePoints);
+				hash.update(str);
+				reference.update(str);
+				const result = hash.digest("hex");
+				expect(result).toMatch(regExp);
+				const expected = reference.digest("hex");
+				expect(result).toBe(expected);
+			});
+		};
+
+		const uncodeRangeTest = (name, start, end) => {
+			const codePoints = [];
+			for (let i = start; i <= end; i++) {
+				codePoints.push(i);
+			}
+			unicodeTest(name, codePoints);
+		};
+
+		uncodeRangeTest("Latin-1 Supplement", 0xa0, 0xff);
+		uncodeRangeTest("Latin Extended", 0x100, 0x24f);
+		uncodeRangeTest("Thaana", 0x780, 0x7bf);
+		uncodeRangeTest("Devanagari", 0x900, 0x97f);
+		uncodeRangeTest("Emoticons", 0x1f600, 0x1f64f);
+
+		unicodeTest("with zero char", "abc\0💩");
+		unicodeTest("weird code point after long code point", [1497, 243248]);
+
+		for (let i = 0; i < 1000; i++) {
+			const chars = Array.from({ length: 20 }, () =>
+				Math.floor(Math.random() * 0x10ffff)
+			);
+			unicodeTest(`fuzzy ${JSON.stringify(chars)}`, chars);
+		}
 	});
 }

yarn.lock

@@ -1208,9 +1208,9 @@ acorn-globals@^6.0.0:
     acorn-walk "^7.1.1"
 
 acorn-import-assertions@^1.7.6:
-  version "1.7.6"
-  resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.7.6.tgz#580e3ffcae6770eebeec76c3b9723201e9d01f78"
-  integrity sha512-FlVvVFA1TX6l3lp8VjDnYYq7R1nyW6x3svAt4nDgrWQ9SBaSh9CnbwgSUTasgfNfOG5HlM1ehugCvM+hjo56LA==
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz#ba2b5939ce62c238db6d93d81c9b111b29b855e9"
+  integrity sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw==
 
 acorn-jsx@^5.3.1:
   version "5.3.1"
@@ -1774,9 +1774,9 @@ coffee-loader@^1.0.0:
     schema-utils "^3.0.0"
 
 coffeescript@^2.5.1:
-  version "2.6.0"
-  resolved "https://registry.yarnpkg.com/coffeescript/-/coffeescript-2.6.0.tgz#927d52aa03df17d445c93c1afb66b081d26e1fa0"
-  integrity sha512-gCGXhR72sTAdEr+oZh3FcOj04DrcMc9lZYSJUBNudkQ4tQXuPKE3cvcYVbK/HiVW+zFzLmnZdHexuJ33ufLZOg==
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/coffeescript/-/coffeescript-2.6.1.tgz#f9e5d4930e1b8a1c5cfba7f95eebd18694ce58fd"
+  integrity sha512-GG5nkF93qII8HmHqnnibkgpp/SV7PSnSPiWsbinwya7nNOe95aE/x2xrKZJFks8Qpko3TNrC+/LahaKgrz5YCg==
 
 collect-v8-coverage@^1.0.0:
   version "1.0.1"
@@ -2290,9 +2290,9 @@ error-ex@^1.3.1:
     is-arrayish "^0.2.1"
 
 es-module-lexer@*, es-module-lexer@^0.9.0:
-  version "0.9.0"
-  resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-0.9.0.tgz#fe4c4621977bc668e285c5f1f70ca3b451095fda"
-  integrity sha512-qU2eN/XHsrl3E4y7mK1wdWnyy5c8gXtCbfP6Xcsemm7fPUR1PIV1JhZfP7ojcN0Fzp69CfrS3u76h2tusvfKiQ==
+  version "0.9.3"
+  resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-0.9.3.tgz#6f13db00cc38417137daf74366f535c8eb438f19"
+  integrity sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ==
 
 es5-ext@^0.10.35, es5-ext@^0.10.45, es5-ext@^0.10.46, es5-ext@^0.10.50, es5-ext@^0.10.51, es5-ext@^0.10.53, es5-ext@~0.10.14, es5-ext@~0.10.2, es5-ext@~0.10.46:
   version "0.10.53"