chore: initial work on benchmarks (#19436)

.github/workflows/test.yml

@@ -60,6 +60,39 @@ jobs:
       # Remove `devDependencies` from `package.json` to avoid `yarn install` compatibility error
       - run: node -e "const content = require('./package.json');delete content.devDependencies;require('fs').writeFileSync('package.json', JSON.stringify(content, null, 2));"
       - run: yarn install --production --frozen-lockfile
+  benchmark:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+          fetch-depth: 0
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+          cache: "yarn"
+      - run: yarn --frozen-lockfile
+      - run: yarn link --frozen-lockfile || true
+      - run: yarn link webpack --frozen-lockfile
+      - run: yarn benchmark --ci
+      - name: Find Comment
+        uses: peter-evans/find-comment@v3
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: "github-actions[bot]"
+          body-includes: Benchmarks
+      - name: Create or update comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body-path: "./test/js/benchmark.md"
+          edit-mode: replace
   basic:
     runs-on: ubuntu-latest
     steps:

.prettierignore

@@ -3,6 +3,7 @@ package.json
 # Ignore some test files
 test/**/*.*
 !test/*.js
+!test/**/*.benchmark.mjs
 !test/**/webpack.config.js
 !test/**/test.config.js
 !test/**/test.filter.js

cspell.json

@@ -170,6 +170,7 @@
     "nosource",
     "nosources",
     "nwjs",
+    "oneline",
     "onconnect",
     "opencollective",
     "opensource",

package.json

@@ -167,7 +167,7 @@
     "pretty-lint": "yarn pretty-lint-base --check",
     "yarn-lint": "yarn-deduplicate --fail --list -s highest yarn.lock",
     "yarn-lint-fix": "yarn-deduplicate -s highest yarn.lock",
-    "benchmark": "node --max-old-space-size=4096 --experimental-vm-modules --trace-deprecation node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/test/*.benchmark.js\" --runInBand",
+    "benchmark": "node --max-old-space-size=4096 --experimental-vm-modules --trace-deprecation node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/test/*.benchmark.mjs\" --runInBand",
     "cover": "yarn cover:all && yarn cover:report",
     "cover:clean": "rimraf .nyc_output coverage",
     "cover:all": "node --expose-gc --max-old-space-size=4096 --experimental-vm-modules node_modules/jest-cli/bin/jest --logHeapUsage --coverage",

test/BenchmarkTestCases.benchmark.js

@@ -1,318 +0,0 @@
-"use strict";
-
-const path = require("path");
-const fs = require("graceful-fs");
-const asyncLib = require("neo-async");
-const Benchmark = require("benchmark");
-const { remove } = require("./helpers/remove");
-
-describe("BenchmarkTestCases", function () {
-	const casesPath = path.join(__dirname, "benchmarkCases");
-	const tests = fs.readdirSync(casesPath).filter(function (folder) {
-		return (
-			!folder.includes("_") &&
-			fs.existsSync(path.resolve(casesPath, folder, "webpack.config.js"))
-		);
-	});
-
-	const baselinesPath = path.join(__dirname, "js", "benchmark-baselines");
-	const baselines = [];
-
-	try {
-		fs.mkdirSync(path.join(__dirname, "js"));
-	} catch (_err) {} // eslint-disable-line no-empty
-	try {
-		fs.mkdirSync(baselinesPath);
-	} catch (_err) {} // eslint-disable-line no-empty
-
-	beforeAll(function (done) {
-		const git = require("simple-git");
-		const rootPath = path.join(__dirname, "..");
-		getBaselineRevs(rootPath, (err, baselineRevisions) => {
-			if (err) return done(err);
-			asyncLib.eachSeries(
-				baselineRevisions,
-				(baselineInfo, callback) => {
-					const baselineRevision = baselineInfo.rev;
-					const baselinePath = path.resolve(baselinesPath, baselineRevision);
-					if (fs.existsSync(path.resolve(baselinePath, ".git"))) {
-						doLoadWebpack();
-					} else {
-						try {
-							fs.mkdirSync(baselinePath);
-						} catch (_err) {} // eslint-disable-line no-empty
-						const gitIndex = path.resolve(rootPath, ".git/index");
-						const index = fs.readFileSync(gitIndex);
-						git(rootPath).raw(
-							["rev-list", "-n", "1", "HEAD"],
-							(err, prevHead) => {
-								if (err) return callback(err);
-								git(baselinePath).raw(
-									[
-										"--git-dir",
-										path.join(rootPath, ".git"),
-										"reset",
-										"--hard",
-										baselineRevision
-									],
-									err => {
-										if (err) return callback(err);
-										git(rootPath).raw(
-											["reset", "--soft", prevHead.split("\n")[0]],
-											err => {
-												if (err) return callback(err);
-												fs.writeFileSync(gitIndex, index);
-												try {
-													doLoadWebpack();
-												} catch (err) {
-													callback(err);
-												}
-											}
-										);
-									}
-								);
-							}
-						);
-					}
-
-					function doLoadWebpack() {
-						const baselineWebpack = jest.requireActual(
-							path.resolve(baselinePath, "lib/index.js")
-						);
-						baselines.push({
-							name: baselineInfo.name,
-							rev: baselineRevision,
-							webpack: baselineWebpack
-						});
-						callback();
-					}
-				},
-				err => {
-					if (err) {
-						done(err);
-						return;
-					}
-					createTests();
-					done();
-				}
-			);
-		});
-	}, 270000);
-
-	afterAll(() => {
-		remove(baselinesPath);
-	});
-
-	function getBaselineRevs(rootPath, callback) {
-		const git = require("simple-git")(rootPath);
-		const lastVersionTag = `v${require("../package.json").version}`;
-		git.raw(["rev-list", "-n", "1", lastVersionTag], (err, resultVersion) => {
-			if (err) return callback(err);
-			const matchVersion = /^([a-f0-9]+)\s*$/.exec(resultVersion);
-			if (!matchVersion)
-				return callback(new Error("Invalid result from git revparse"));
-			const revLastVersion = matchVersion[1];
-			git.raw(
-				["rev-list", "--parents", "-n", "1", "HEAD"],
-				(err, resultParents) => {
-					if (err) return callback(err);
-					const match = /^([a-f0-9]+)\s*([a-f0-9]+)\s*([a-f0-9]+)?\s*$/.exec(
-						resultParents
-					);
-					if (!match)
-						return callback(new Error("Invalid result from git rev-list"));
-					const head = match[1];
-					const parent1 = match[2];
-					const parent2 = match[3];
-					if (parent2 && parent1) {
-						return callback(
-							null,
-							[
-								{
-									name: "HEAD",
-									rev: head
-								},
-								head !== revLastVersion && {
-									name: lastVersionTag,
-									rev: revLastVersion
-								},
-								parent1 !== revLastVersion &&
-									head !== revLastVersion && {
-										name: "base",
-										rev: parent1
-									}
-							].filter(Boolean)
-						);
-					} else if (parent1) {
-						return callback(
-							null,
-							[
-								{
-									name: "HEAD",
-									rev: head
-								},
-								head !== revLastVersion && {
-									name: lastVersionTag,
-									rev: revLastVersion
-								}
-							].filter(Boolean)
-						);
-					}
-					return callback(new Error("No baseline found"));
-				}
-			);
-		});
-	}
-
-	function tDistribution(n) {
-		// two-sided, 90%
-		// https://en.wikipedia.org/wiki/Student%27s_t-distribution
-		if (n <= 30) {
-			//            1      2      ...
-			const data = [
-				6.314, 2.92, 2.353, 2.132, 2.015, 1.943, 1.895, 1.86, 1.833, 1.812,
-				1.796, 1.782, 1.771, 1.761, 1.753, 1.746, 1.74, 1.734, 1.729, 1.725,
-				1.721, 1.717, 1.714, 1.711, 1.708, 1.706, 1.703, 1.701, 1.699, 1.697
-			];
-			return data[n - 1];
-		} else if (n <= 120) {
-			//            30     40     50     60     70     80     90     100    110    120
-			const data = [
-				1.697, 1.684, 1.676, 1.671, 1.667, 1.664, 1.662, 1.66, 1.659, 1.658
-			];
-			var a = data[Math.floor(n / 10) - 3];
-			var b = data[Math.ceil(n / 10) - 3];
-			var f = n / 10 - Math.floor(n / 10);
-			return a * (1 - f) + b * f;
-		}
-
-		return 1.645;
-	}
-
-	function runBenchmark(webpack, config, callback) {
-		// warmup
-		const warmupCompiler = webpack(config, (err, stats) => {
-			warmupCompiler.purgeInputFileSystem();
-			const bench = new Benchmark(
-				function (deferred) {
-					const compiler = webpack(config, (err, stats) => {
-						compiler.purgeInputFileSystem();
-						if (err) {
-							callback(err);
-							return;
-						}
-						if (stats.hasErrors()) {
-							callback(new Error(stats.toJson().errors.join("\n\n")));
-							return;
-						}
-						deferred.resolve();
-					});
-				},
-				{
-					maxTime: 30,
-					defer: true,
-					initCount: 1,
-					onComplete: function () {
-						const stats = bench.stats;
-						const n = stats.sample.length;
-						const nSqrt = Math.sqrt(n);
-						const z = tDistribution(n - 1);
-						stats.minConfidence = stats.mean - (z * stats.deviation) / nSqrt;
-						stats.maxConfidence = stats.mean + (z * stats.deviation) / nSqrt;
-						stats.text = `${Math.round(stats.mean * 1000)} ms ± ${Math.round(
-							stats.deviation * 1000
-						)} ms [${Math.round(stats.minConfidence * 1000)} ms; ${Math.round(
-							stats.maxConfidence * 1000
-						)} ms]`;
-						callback(null, bench.stats);
-					},
-					onError: callback
-				}
-			);
-			bench.run({
-				async: true
-			});
-		});
-	}
-
-	function createTests() {
-		for (const testName of tests) {
-			const testDirectory = path.join(casesPath, testName);
-			let headStats = null;
-			describe(`${testName} create benchmarks`, function () {
-				for (const baseline of baselines) {
-					let baselineStats = null;
-					// eslint-disable-next-line no-loop-func
-					it(`should benchmark ${baseline.name} (${baseline.rev})`, function (done) {
-						const outputDirectory = path.join(
-							__dirname,
-							"js",
-							"benchmark",
-							`baseline-${baseline.name}`,
-							testName
-						);
-						const config =
-							Object.create(
-								jest.requireActual(
-									path.join(testDirectory, "webpack.config.js")
-								)
-							) || {};
-						config.output = Object.create(config.output || {});
-						if (!config.context) config.context = testDirectory;
-						if (!config.output.path) config.output.path = outputDirectory;
-						runBenchmark(baseline.webpack, config, (err, stats) => {
-							if (err) return done(err);
-							process.stderr.write(`        ${baseline.name} ${stats.text}`);
-							if (baseline.name === "HEAD") headStats = stats;
-							else baselineStats = stats;
-							done();
-						});
-					}, 180000);
-					// eslint-disable-next-line no-loop-func
-					it(`should benchmark ${baseline.name} (${baseline.rev})`, done => {
-						const outputDirectory = path.join(
-							__dirname,
-							"js",
-							"benchmark",
-							`baseline-${baseline.name}`,
-							testName
-						);
-						const config =
-							jest.requireActual(
-								path.join(testDirectory, "webpack.config.js")
-							) || {};
-						config.output = config.output || {};
-						if (!config.context) config.context = testDirectory;
-						if (!config.output.path) config.output.path = outputDirectory;
-						runBenchmark(baseline.webpack, config, (err, stats) => {
-							if (err) return done(err);
-							process.stderr.write(`        ${baseline.name} ${stats.text}`);
-							if (baseline.name === "HEAD") headStats = stats;
-							else baselineStats = stats;
-							done();
-						});
-					}, 180000);
-
-					if (baseline.name !== "HEAD") {
-						// eslint-disable-next-line no-loop-func
-						it(`HEAD should not be slower than ${baseline.name} (${baseline.rev})`, function () {
-							if (baselineStats.maxConfidence < headStats.minConfidence) {
-								throw new Error(
-									`HEAD (${headStats.text}) is slower than ${baseline.name} (${baselineStats.text}) (90% confidence)`
-								);
-							} else if (
-								baselineStats.minConfidence > headStats.maxConfidence
-							) {
-								console.log(
-									`======> HEAD is ${Math.round(
-										(baselineStats.mean / headStats.mean) * 100 - 100
-									)}% faster than ${baseline.name} (90% confidence)!`
-								);
-							}
-						});
-					}
-				}
-			});
-		}
-	}
-});

test/BenchmarkTestCases.benchmark.mjs

@@ -0,0 +1,317 @@
+"use strict";
+
+import path from "path";
+import fs from "fs/promises";
+import { createWriteStream } from "fs";
+import Benchmark from "benchmark";
+import { remove } from "./helpers/remove";
+import { dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import simpleGit from "simple-git";
+import { jest } from "@jest/globals";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const rootPath = path.join(__dirname, "..");
+const git = simpleGit(rootPath);
+
+const REV_LIST_REGEXP = /^([a-f0-9]+)\s*([a-f0-9]+)\s*([a-f0-9]+)?\s*$/;
+
+async function getHead(revList) {
+	if (typeof process.env.HEAD !== "undefined") {
+		return process.env.HEAD;
+	}
+
+	if (revList[3]) {
+		return revList[3];
+	}
+
+	return revList[1];
+}
+
+async function getBase(revList) {
+	if (typeof process.env.BASE !== "undefined") {
+		return process.env.BASE;
+	}
+
+	if (revList[3]) {
+		return revList[2];
+	}
+
+	const branchName = await git.raw(["rev-parse", "--abbrev-ref", "HEAD"]);
+
+	if (branchName !== "main") {
+		const resultParents = await git.raw([
+			"rev-list",
+			"--parents",
+			"-n",
+			"1",
+			"main"
+		]);
+
+		const revList = REV_LIST_REGEXP.exec(resultParents);
+
+		if (!revList[1]) {
+			throw new Error("No parent commit found");
+		}
+
+		return revList[1];
+	}
+
+	return revList[2];
+}
+
+async function getBaselineRevs() {
+	const resultParents = await git.raw([
+		"rev-list",
+		"--parents",
+		"-n",
+		"1",
+		"HEAD"
+	]);
+	const revList = REV_LIST_REGEXP.exec(resultParents);
+
+	if (!revList) throw new Error("Invalid result from git rev-list");
+
+	const head = await getHead(revList);
+	const base = await getBase(revList);
+
+	if (!head || !base) {
+		throw new Error("No baseline found");
+	}
+
+	return [
+		{
+			name: "HEAD",
+			rev: head
+		},
+		{
+			name: "BASE",
+			rev: base
+		}
+	];
+}
+
+function runBenchmark(webpack, config, callback) {
+	// warmup
+	const warmupCompiler = webpack(config, (err, stats) => {
+		if (err) {
+			callback(err);
+			return;
+		}
+
+		warmupCompiler.purgeInputFileSystem();
+
+		const bench = new Benchmark(
+			function (deferred) {
+				const compiler = webpack(config, (err, stats) => {
+					compiler.purgeInputFileSystem();
+					if (err) {
+						callback(err);
+						return;
+					}
+
+					if (stats.hasErrors()) {
+						callback(new Error(stats.toString()));
+						return;
+					}
+
+					deferred.resolve();
+				});
+			},
+			{
+				maxTime: 30,
+				defer: true,
+				initCount: 1,
+				onComplete: function () {
+					const stats = bench.stats;
+					const n = stats.sample.length;
+					const nSqrt = Math.sqrt(n);
+					const z = tDistribution(n - 1);
+
+					stats.sampleCount = stats.sample.length;
+					stats.minConfidence = stats.mean - (z * stats.deviation) / nSqrt;
+					stats.maxConfidence = stats.mean + (z * stats.deviation) / nSqrt;
+					stats.text = `${Math.round(stats.mean * 1000)} ms ± ${Math.round(
+						stats.deviation * 1000
+					)} ms [${Math.round(stats.minConfidence * 1000)} ms; ${Math.round(
+						stats.maxConfidence * 1000
+					)} ms]`;
+
+					callback(null, bench.stats);
+				},
+				onError: callback
+			}
+		);
+
+		bench.run({
+			async: true
+		});
+	});
+}
+
+function tDistribution(n) {
+	// two-sided, 90%
+	// https://en.wikipedia.org/wiki/Student%27s_t-distribution
+	if (n <= 30) {
+		//            1      2      ...
+		const data = [
+			6.314, 2.92, 2.353, 2.132, 2.015, 1.943, 1.895, 1.86, 1.833, 1.812, 1.796,
+			1.782, 1.771, 1.761, 1.753, 1.746, 1.74, 1.734, 1.729, 1.725, 1.721,
+			1.717, 1.714, 1.711, 1.708, 1.706, 1.703, 1.701, 1.699, 1.697
+		];
+		return data[n - 1];
+	} else if (n <= 120) {
+		//            30     40     50     60     70     80     90     100    110    120
+		const data = [
+			1.697, 1.684, 1.676, 1.671, 1.667, 1.664, 1.662, 1.66, 1.659, 1.658
+		];
+		const a = data[Math.floor(n / 10) - 3];
+		const b = data[Math.ceil(n / 10) - 3];
+		const f = n / 10 - Math.floor(n / 10);
+
+		return a * (1 - f) + b * f;
+	}
+
+	return 1.645;
+}
+
+const casesPath = path.join(__dirname, "benchmarkCases");
+
+const tests = [];
+
+for (const folder of await fs.readdir(casesPath)) {
+	if (folder.includes("_")) {
+		continue;
+	}
+
+	try {
+		await fs.access(
+			path.resolve(casesPath, folder, "webpack.config.js"),
+			fs.constants.R_OK
+		);
+	} catch (e) {
+		continue;
+	}
+
+	tests.push(folder);
+}
+
+const output = path.join(__dirname, "js");
+const baselinesPath = path.join(output, "benchmark-baselines");
+const baselines = [];
+
+try {
+	await fs.mkdir(baselinesPath, { recursive: true });
+} catch (_err) {} // eslint-disable-line no-empty
+
+const baselineRevisions = await getBaselineRevs();
+
+for (const baselineInfo of baselineRevisions) {
+	function doLoadWebpack() {
+		baselines.push({
+			name: baselineInfo.name,
+			rev: baselineRevision,
+			webpack: () => {
+				return jest.requireActual(path.resolve(baselinePath, "lib/index.js"));
+			}
+		});
+	}
+
+	const baselineRevision = baselineInfo.rev;
+	const baselinePath = path.resolve(baselinesPath, baselineRevision);
+
+	try {
+		await fs.access(path.resolve(baselinePath, ".git"), fs.constants.R_OK);
+	} catch (_err) {
+		try {
+			await fs.mkdir(baselinePath);
+		} catch (_err) {} // eslint-disable-line no-empty
+
+		const gitIndex = path.resolve(rootPath, ".git/index");
+		const index = await fs.readFile(gitIndex);
+		const prevHead = await git.raw(["rev-list", "-n", "1", "HEAD"]);
+
+		await simpleGit(baselinePath).raw([
+			"--git-dir",
+			path.join(rootPath, ".git"),
+			"reset",
+			"--hard",
+			baselineRevision
+		]);
+
+		await git.raw(["reset", "--soft", prevHead.split("\n")[0]]);
+		await fs.writeFile(gitIndex, index);
+	} finally {
+		doLoadWebpack();
+	}
+}
+
+const reportFilePath = path.resolve(output, "benchmark.md");
+const report = createWriteStream(reportFilePath, { flags: "w" });
+
+report.write("### Benchmarks:\n\n");
+
+describe("BenchmarkTestCases", function () {
+	for (const testName of tests) {
+		const testDirectory = path.join(casesPath, testName);
+		let headStats = null;
+
+		describe(`${testName} create benchmarks`, function () {
+			for (const baseline of baselines) {
+				let baselineStats = null;
+
+				// eslint-disable-next-line no-loop-func
+				it(`should benchmark ${baseline.name} (${baseline.rev})`, done => {
+					const outputDirectory = path.join(
+						__dirname,
+						"js",
+						"benchmark",
+						`baseline-${baseline.name}`,
+						testName
+					);
+					const config =
+						jest.requireActual(path.join(testDirectory, "webpack.config.js")) ||
+						{};
+
+					config.mode = config.mode || "production";
+					config.output = config.output || {};
+
+					if (!config.context) config.context = testDirectory;
+					if (!config.output.path) config.output.path = outputDirectory;
+					runBenchmark(baseline.webpack(), config, (err, stats) => {
+						if (err) return done(err);
+						report.write(
+							`- "${testName}": ${baseline.name} (${baseline.rev}) ${stats.text} (${stats.sampleCount} runs)\n`
+						);
+						if (baseline.name === "HEAD") headStats = stats;
+						else baselineStats = stats;
+						done();
+					});
+				}, 180000);
+
+				if (baseline.name !== "HEAD") {
+					// eslint-disable-next-line no-loop-func
+					it(`HEAD and ${baseline.name} (${baseline.rev}) results`, function () {
+						if (!baselineStats) {
+							throw new Error("No baseline stats");
+						}
+
+						report.write(`- "${testName}" change: `);
+						report.write(
+							`HEAD (${headStats.text}) is ${Math.round(
+								(baselineStats.mean / headStats.mean) * 100 - 100
+							)}% ${baselineStats.maxConfidence < headStats.minConfidence ? "slower than" : baselineStats.minConfidence > headStats.maxConfidence ? "faster than" : "the same as"} BASE (${baseline.name}) (${baselineStats.text})\n`
+						);
+
+						report.write(`-----\n`);
+					});
+				}
+			}
+		});
+	}
+
+	afterAll(() => {
+		remove(baselinesPath);
+		report.end();
+	});
+});

test/benchmarkCases/devtool-source-map/webpack.config.js

@@ -1,5 +1,5 @@
 /** @type {import("../../../").Configuration} */
 module.exports = {
 	entry: "./index",
-	devtool: "eval-cheap-module-source-map"
+	devtool: "source-map"
 };

test/benchmarkCases/large-ast/index.js

@@ -1,12 +0,0 @@
-import "react/dist/react-with-addons.js?0";
-import "react/dist/react-with-addons.js?1";
-import "react/dist/react-with-addons.js?2";
-import "react/dist/react-with-addons.js?3";
-import "react/dist/react-with-addons.js?4";
-import "react/dist/react-with-addons.js?5";
-import "react/dist/react-with-addons.js?6";
-import "react/dist/react-with-addons.js?7";
-import "react/dist/react-with-addons.js?8";
-import "react/dist/react-with-addons.js?9";
-import "react/dist/react-with-addons.js?10";
-import "react/dist/react-with-addons.js?11";

test/benchmarkCases/large-ast/index2.js

@@ -1,8 +0,0 @@
-import "lodash/lodash.js?0";
-import "lodash/lodash.js?1";
-import "lodash/lodash.js?2";
-import "lodash/lodash.js?3";
-import "lodash/lodash.min.js?0";
-import "lodash/lodash.min.js?1";
-import "lodash/lodash.min.js?2";
-import "lodash/lodash.min.js?3";

test/benchmarkCases/large-ast/webpack.config.js

@@ -1,4 +0,0 @@
-/** @type {import("../../../").Configuration} */
-module.exports = {
-	entry: ["./index", "./index2"]
-};

test/benchmarkCases/many-stuff-harmony/a.js

@@ -1,11 +0,0 @@
-module.exports = function() {
-	let str = "";
-	let sum = ["1"];
-	const query = +this.query.slice(1);
-	for(let i = 0; i < query; i++) {
-		str += `import b${i} from "./b?${Math.floor(i/2)}!";\n`;
-		sum.push(`b${i}`);
-	}
-	str += "export default " + sum.join(" + ");
-	return str;
-}